Обсуждение: Remove page-read callback from XLogReaderState.
Hello. As mentioned before [1], read_page callback in XLogReaderState is a cause of headaches. Adding another remote-controlling stuff to xlog readers makes things messier [2]. I refactored XLOG reading functions so that we don't need the callback. In short, ReadRecrod now calls XLogPageRead directly with the attached patch set. | while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) | == XLREAD_NEED_DATA) | XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); On the other hand, XLogReadRecord became a bit complex. The patch makes XLogReadRecord a state machine. I'm not confident that the additional complexity is worth doing. Anyway I'll gegister this to the next CF. [1] https://www.postgresql.org/message-id/47215279-228d-f30d-35d1-16af695e53f3@iki.fi [2] https://www.postgresql.org/message-id/20190412.122711.158276916.horiguchi.kyotaro@lab.ntt.co.jp regards. -- Kyotaro Horiguchi NTT Open Source Software Center From 3de14fb47987e9dd8189bfad3ca7264c26b719eb Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 10:22:49 +0900 Subject: [PATCH 01/10] Define macros to make XLogReadRecord a state machine To minimize apparent imapct on code, use some macros as syntax sugar. This is a similar stuff with ExecInterpExpr but a bitdifferent. The most significant difference is that this stuff allows some functions are leaved midst of their work thencontinue. Roughly speaking this is used as the follows. enum retval some_func() { static .. internal_variables; XLR_SWITCH(); ... XLR_LEAVE(STATUS1, RETVAL_CONTINUE); ... XLR_LEAVE(STATUS2, RETVAL_CONTINUE2); ... XLR_SWITCH_END(); XLR_RETURN(RETVAL_FINISH); } The caller uses the function as follows: while (some_func() != RETVAL_FINISH) { <do some work>; } --- src/backend/access/transam/xlogreader.c | 63 +++++++++++++++++++++++++++++++++ src/include/access/xlogreader.h | 3 ++ 2 files changed, 66 insertions(+) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 9196aa3aae..5299765040 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -29,6 +29,69 @@ #include "utils/memutils.h" #endif +/* + * Use computed-goto-based opcode dispatch when computed gotos are available. + * But use a separate symbol so that it's easy to adjust locally in this file + * for development and testing. + */ +#ifdef HAVE_COMPUTED_GOTO +#define XLR_USE_COMPUTED_GOTO +#endif /* HAVE_COMPUTED_GOTO */ + +/* + * Macros for opcode dispatch. + * + * XLR_SWITCH - just hides the switch if not in use. + * XLR_CASE - labels the implementation of named expression step type. + * XLR_DISPATCH - jump to the implementation of the step type for 'op'. + * XLR_LEAVE - leave the function and return here at the next call. + * XLR_RETURN - return from the function and set state to initial state. + * XLR_END - just hides the closing brace if not in use. + */ +#if defined(XLR_USE_COMPUTED_GOTO) +#define XLR_SWITCH() \ + /* Don't call duplicatedly */ \ + static int callcnt = 0 PG_USED_FOR_ASSERTS_ONLY; \ + do { \ + if ((XLR_STATE).j) \ + goto *((void *) (XLR_STATE).j); \ + XLR_CASE(XLR_INIT_STATE); \ + Assert(++callcnt == 1); \ + } while (0) +#define XLR_CASE(name) name: +#define XLR_DISPATCH() goto *((void *) (XLR_STATE).j) +#define XLR_LEAVE(name, code) do { \ + (XLR_STATE).j = (&&name); return (code); \ + XLR_CASE(name); \ + } while (0) +#define XLR_RETURN(code) \ + do { \ + Assert(--callcnt == 0); \ + (XLR_STATE).j = (&&XLR_INIT_STATE); return (code); \ + } while (0) +#define XLR_SWITCH_END() +#else /* !XLR_USE_COMPUTED_GOTO */ +#define XLR_SWITCH() \ + /* Don't call duplicatedly */ \ + static int callcnt = 0 PG_USED_FOR_ASSERTS_ONLY; \ + switch ((XLR_STATE).c) { \ + XLR_CASE(XLR_INIT_STATE); \ + Assert(++callcnt == 1); \ +#define XLR_CASE(name) case name: +#define XLR_DISPATCH() goto starteval +#define XLR_LEAVE(name, code) \ + do { \ + (XLR_STATE).c = (name); return (code); \ + XLR_CASE(name); \ + } while (0) +#define XLR_RETURN(code) \ + do { \ + Assert(--callcnt == 0); \ + (XLR_STATE).c = (XLR_INIT_STATE); return (code); \ + } while (0) +#define XLR_SWITCH_END() } +#endif /* XLR_USE_COMPUTED_GOTO */ + static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f3bae0bf49..30500c35c7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -240,6 +240,9 @@ extern bool DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, #define XLogRecBlockImageApply(decoder, block_id) \ ((decoder)->blocks[block_id].apply_image) +/* Reset the reader state */ +#define XLREAD_RESET(state) ((state)->xlnd_state.j = (state)->xlread_state.j = 0) + extern bool RestoreBlockImage(XLogReaderState *recoder, uint8 block_id, char *dst); extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len); extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, -- 2.16.3 From 59675513983b70fc1f71c7770b345553a0320c23 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 14:15:16 +0900 Subject: [PATCH 02/10] Make ReadPageInternal a state machine This patch set aims to remove read_page call back from XLogReaderState. This is done in two steps. This patch does the first stap. Makes ReadPageInternal, which currently calling read_page callback, a state machine and move the caller sites to XLogReadRecord and other direct callers of the callback. --- src/backend/access/transam/xlog.c | 15 ++- src/backend/access/transam/xlogreader.c | 175 +++++++++++++++---------- src/backend/access/transam/xlogutils.c | 8 +- src/backend/replication/logical/logicalfuncs.c | 6 +- src/backend/replication/walsender.c | 10 +- src/bin/pg_rewind/parsexlog.c | 17 ++- src/bin/pg_waldump/pg_waldump.c | 8 +- src/include/access/xlogreader.h | 46 +++++-- src/include/access/xlogutils.h | 2 +- src/include/replication/logicalfuncs.h | 2 +- 10 files changed, 181 insertions(+), 108 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c00b63c751..2130671d36 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -885,7 +885,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, +static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, @@ -11507,7 +11507,7 @@ CancelBackup(void) * XLogPageRead() to try fetching the record from another source, or to * sleep and retry. */ -static int +static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI) { @@ -11566,7 +11566,8 @@ retry: readLen = 0; readSource = 0; - return -1; + xlogreader->readLen = -1; + return; } } @@ -11661,7 +11662,8 @@ retry: goto next_record_is_invalid; } - return readLen; + xlogreader->readLen = readLen; + return; next_record_is_invalid: lastSourceFailed = true; @@ -11675,8 +11677,9 @@ next_record_is_invalid: /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - else - return -1; + + xlogreader->readLen = -1; + return; } /* diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 5299765040..f8375eceeb 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -98,8 +98,7 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr); -static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, - int reqLen); +static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen); static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static void ResetDecoder(XLogReaderState *state); @@ -286,7 +285,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) uint32 targetRecOff; uint32 pageHeaderSize; bool gotheader; - int readOff; /* * randAccess indicates whether to verify the previous-record pointer of @@ -338,15 +336,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) * byte to cover the whole record header, or at least the part of it that * fits on the same page. */ - readOff = ReadPageInternal(state, - targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* - * ReadPageInternal always returns at least the page header, so we can - * examine it now. + * We have loaded at least the page header, so we can examine it now. */ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); if (targetRecOff == 0) @@ -372,8 +372,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) goto err; } - /* ReadPageInternal has verified the page header */ - Assert(pageHeaderSize <= readOff); + /* XLogNeedData has verified the page header */ + Assert(pageHeaderSize <= state->readLen); /* * Read the record length. @@ -450,14 +450,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) targetPagePtr += XLOG_BLCKSZ; /* Wait for the next page to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(total_len - gotlen + SizeOfXLogShortPHD, - XLOG_BLCKSZ)); + while (XLogNeedData(state, targetPagePtr, + Min(total_len - gotlen + SizeOfXLogShortPHD, + XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); - if (readOff < 0) + if (state->readLen < 0) goto err; - Assert(SizeOfXLogShortPHD <= readOff); + Assert(SizeOfXLogShortPHD <= state->readLen); /* Check that the continuation on next page looks valid */ pageHeader = (XLogPageHeader) state->readBuf; @@ -486,20 +489,28 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* Append the continuation from this page to the buffer */ pageHeaderSize = XLogPageHeaderSize(pageHeader); - if (readOff < pageHeaderSize) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize); + if (state->readLen < pageHeaderSize) + { + while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + } - Assert(pageHeaderSize <= readOff); + Assert(pageHeaderSize <= state->readLen); contdata = (char *) state->readBuf + pageHeaderSize; len = XLOG_BLCKSZ - pageHeaderSize; if (pageHeader->xlp_rem_len < len) len = pageHeader->xlp_rem_len; - if (readOff < pageHeaderSize + len) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize + len); + if (state->readLen < pageHeaderSize + len) + { + if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + } memcpy(buffer, (char *) contdata, len); buffer += len; @@ -530,9 +541,13 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) else { /* Wait for the record data to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + total_len, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + total_len, XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* Record does not cross a page boundary */ @@ -575,32 +590,41 @@ err: } /* - * Read a single xlog page including at least [pageptr, reqLen] of valid data - * via the read_page() callback. + * Checks that an xlog page loaded in state->readBuf is including at least + * [pageptr, reqLen] and the page is valid. * - * Returns -1 if the required page cannot be read for some reason; errormsg_buf - * is set in that case (unless the error occurs in the read_page callback). + * Returns false if the required data is fully loaded. state->readLen is set to + * -1 when the loaded data is found to be invalid. * - * We fetch the page from a reader-local cache if we know we have the required - * data and if there hasn't been any error since caching the data. + * Otherwise, returns true and requests data using state->loadPagePtr and + * state->loadLen. The caller should load the region to state->readBuf and + * call this function again. */ -static int -ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) +static bool +XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) { - int readLen; - uint32 targetPageOff; - XLogSegNo targetSegNo; - XLogPageHeader hdr; + /* + * This function is a state machine that can exit and reenter at any place + * marked as XLR_LEAVE. All internal state is preserved through multiple + * calls. + */ + static uint32 targetPageOff; + static XLogSegNo targetSegNo; + static XLogPageHeader hdr; - Assert((pageptr % XLOG_BLCKSZ) == 0); +#define XLR_STATE state->xlnd_state +#define XLR_INIT_STATE XLND_STATE_INIT + + XLR_SWITCH (); XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size); targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size); + Assert((pageptr % XLOG_BLCKSZ) == 0); /* check whether we have all the requested data already */ if (targetSegNo == state->readSegNo && targetPageOff == state->readOff && reqLen <= state->readLen) - return state->readLen; + XLR_RETURN(false); /* * Data is not in our buffer. @@ -616,18 +640,17 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) */ if (targetSegNo != state->readSegNo && targetPageOff != 0) { - XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; + state->loadPagePtr = pageptr - targetPageOff; + state->loadLen = XLOG_BLCKSZ; + XLR_LEAVE(XLND_STATE_SEGHEAD, true); - readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ, - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + if (state->readLen < 0) goto err; /* we can be sure to have enough WAL available, we scrolled back */ - Assert(readLen == XLOG_BLCKSZ); + Assert(state->readLen == XLOG_BLCKSZ); - if (!XLogReaderValidatePageHeader(state, targetSegmentPtr, + if (!XLogReaderValidatePageHeader(state, state->loadPagePtr, state->readBuf)) goto err; } @@ -636,48 +659,53 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) * First, read the requested data length, but at least a short page header * so that we can validate it. */ - readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + state->loadPagePtr = pageptr; + state->loadLen = Max(reqLen, SizeOfXLogShortPHD); + XLR_LEAVE(XLND_STATE_PAGEHEAD, true); + + if (state->readLen < 0) goto err; - Assert(readLen <= XLOG_BLCKSZ); + Assert(state->readLen <= XLOG_BLCKSZ); /* Do we have enough data to check the header length? */ - if (readLen <= SizeOfXLogShortPHD) + if (state->readLen <= SizeOfXLogShortPHD) goto err; - Assert(readLen >= reqLen); + Assert(state->readLen >= state->loadLen); hdr = (XLogPageHeader) state->readBuf; /* still not enough */ - if (readLen < XLogPageHeaderSize(hdr)) + if (state->readLen < XLogPageHeaderSize(hdr)) { - readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr), - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + state->loadPagePtr = pageptr; + state->loadLen = XLogPageHeaderSize(hdr); + XLR_LEAVE(XLND_STATE_PAGEFULLHEAD, true); + + if (state->readLen < 0) goto err; } + XLR_SWITCH_END(); + /* * Now that we know we have the full header, validate it. */ - if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr)) - goto err; + if (!XLogReaderValidatePageHeader(state, pageptr, (char *) state->readBuf)) + goto err; /* update read state information */ state->readSegNo = targetSegNo; state->readOff = targetPageOff; - state->readLen = readLen; - return readLen; + XLR_RETURN(false); err: XLogReaderInvalReadState(state); - return -1; + state->readLen = -1; + + XLR_RETURN(false); } /* @@ -976,7 +1004,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr targetPagePtr; int targetRecOff; uint32 pageHeaderSize; - int readLen; /* * Compute targetRecOff. It should typically be equal or greater than @@ -984,7 +1011,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * that, except when caller has explicitly specified the offset that * falls somewhere there or when we are skipping multi-page * continuation record. It doesn't matter though because - * ReadPageInternal() is prepared to handle that and will read at + * CheckPage() is prepared to handle that and will read at * least short page-header worth of data */ targetRecOff = tmpRecPtr % XLOG_BLCKSZ; @@ -993,8 +1020,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) targetPagePtr = tmpRecPtr - targetRecOff; /* Read the page containing the record */ - readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); - if (readLen < 0) + while(XLogNeedData(state, targetPagePtr, targetRecOff)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; header = (XLogPageHeader) state->readBuf; @@ -1002,8 +1033,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) pageHeaderSize = XLogPageHeaderSize(header); /* make sure we have enough data for the page header */ - readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); - if (readLen < 0) + while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* skip over potential continuation data */ diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 10a663bae6..c853e1f0e3 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * exists for normal backends, so we have to do a check/sleep/repeat style of * loop for now. */ -int +void read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) @@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, else if (targetPagePtr + reqLen > read_upto) { /* not enough data there */ - return -1; + state->readLen = -1; + return; } else { @@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, XLOG_BLCKSZ); /* number of valid bytes in the buffer */ - return count; + state->readLen = count; + return; } diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index d974400d6e..16c6095179 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -114,12 +114,12 @@ check_permissions(void) (errmsg("must be superuser or replication role to use replication slots")))); } -int +void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { - return read_local_xlog_page(state, targetPagePtr, reqLen, - targetRecPtr, cur_page, pageTLI); + read_local_xlog_page(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, pageTLI); } /* diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 440b6aac4b..1ef952b39a 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd) * which has to do a plain sleep/busy loop, because the walsender's latch gets * set every time WAL is flushed. */ -static int +static void logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { @@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* fail if not (implies we are going to shut down) */ if (flushptr < targetPagePtr + reqLen) - return -1; + { + state->readLen = -1; + return; + } if (targetPagePtr + XLOG_BLCKSZ <= flushptr) count = XLOG_BLCKSZ; /* more than one block available */ @@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* now actually read the data, we know it's there */ XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ); - return count; + state->readLen = count; + return; } /* diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 7709b96e00..c3b9e738f7 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -48,7 +48,7 @@ typedef struct XLogPageReadPrivate int tliIndex; } XLogPageReadPrivate; -static int SimpleXLogPageRead(XLogReaderState *xlogreader, +static void SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); @@ -237,7 +237,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, } /* XLogreader callback function, to read a WAL page */ -static int +static void SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) @@ -292,7 +292,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (xlogreadfd < 0) { pg_log_error("could not open file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return; } } @@ -305,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) { pg_log_error("could not seek in file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return; } @@ -318,13 +320,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, pg_log_error("could not read file \"%s\": read %d of %zu", xlogfpath, r, (Size) XLOG_BLCKSZ); - return -1; + xlogreader->readLen = -1; + return; } Assert(targetSegNo == xlogreadsegno); *pageTLI = targetHistory[private->tliIndex].tli; - return XLOG_BLCKSZ; + + xlogreader->readLen = XLOG_BLCKSZ; + return; } /* diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index e106fb2ed1..9ad70a2f5c 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, /* * XLogReader read_page callback */ -static int +static void XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI) { @@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, else { private->endptr_reached = true; - return -1; + state->readLen = -1; + return; } } XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr, readBuff, count); - return count; + state->readLen = count; + return; } /* diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 30500c35c7..268252e88f 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -30,7 +30,7 @@ typedef struct XLogReaderState XLogReaderState; /* Function type definition for the read_page callback */ -typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader, +typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, @@ -66,6 +66,15 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* internal state of XLogNeedData() */ +typedef enum xlnd_stateid +{ + XLND_STATE_INIT, + XLND_STATE_SEGHEAD, + XLND_STATE_PAGEHEAD, + XLND_STATE_PAGEFULLHEAD +} xlnd_stateid; + struct XLogReaderState { /* ---------------------------------------- @@ -120,6 +129,22 @@ struct XLogReaderState XLogRecPtr EndRecPtr; /* end+1 of last record read */ + /* ---------------------------------------- + * Communication with page reader + * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes. + * ---------------------------------------- + */ + /* parameters to page reader */ + XLogRecPtr loadPagePtr; /* Pointer to the page */ + int loadLen; /* wanted length in bytes */ + char *readBuf; /* buffer to store data */ + XLogRecPtr currRecPtr; /* beginning of the WAL record being read */ + + /* return from page reader */ + int32 readLen; /* bytes acutually read, must be larger than + * loadLen. -1 on error. */ + TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + /* ---------------------------------------- * Decoded representation of current record * @@ -145,17 +170,9 @@ struct XLogReaderState * ---------------------------------------- */ - /* - * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least - * readLen bytes) - */ - char *readBuf; - uint32 readLen; - - /* last read segment, segment offset, TLI for data currently in readBuf */ + /* last read segment and segment offset for data currently in readBuf */ XLogSegNo readSegNo; uint32 readOff; - TimeLineID readPageTLI; /* * beginning of prior page read, and its TLI. Doesn't necessarily @@ -164,8 +181,6 @@ struct XLogReaderState XLogRecPtr latestPagePtr; TimeLineID latestPageTLI; - /* beginning of the WAL record being read. */ - XLogRecPtr currRecPtr; /* timeline to read it from, 0 if a lookup is required */ TimeLineID currTLI; @@ -194,6 +209,13 @@ struct XLogReaderState /* Buffer to hold error message */ char *errormsg_buf; + + /* Internal state of XLogNeedData */ + union + { + xlnd_stateid c; + void *j; + } xlnd_state; }; /* Get a new XLogReader */ diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 0ab5ba62f5..5dba86b8b8 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern int read_local_xlog_page(XLogReaderState *state, +extern void read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI); diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h index 3fb7ad5d67..3d00dee067 100644 --- a/src/include/replication/logicalfuncs.h +++ b/src/include/replication/logicalfuncs.h @@ -11,7 +11,7 @@ #include "replication/logical.h" -extern int logical_read_local_xlog_page(XLogReaderState *state, +extern void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI); -- 2.16.3 From 7ce63df6ca10576ac0c85e64d239f98a71bf6003 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 09:57:07 +0900 Subject: [PATCH 03/10] Change interface of XLogReadRecord As a preparation to the second step, this patch changes the interface of XLogReadRecord so that the function can request the callers for reading more data. --- src/backend/access/transam/twophase.c | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/access/transam/xlogreader.c | 52 ++++++++++++++------------ src/backend/replication/logical/logical.c | 2 +- src/backend/replication/logical/logicalfuncs.c | 2 +- src/backend/replication/slotfuncs.c | 2 +- src/backend/replication/walsender.c | 2 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 11 +++++- 10 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index a399c0052d..5dba27e5dd 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1394,7 +1394,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errmsg("out of memory"), errdetail("Failed while allocating a WAL reading processor."))); - record = XLogReadRecord(xlogreader, lsn, &errormsg); + XLogReadRecord(xlogreader, lsn, &record, &errormsg); if (record == NULL) ereport(ERROR, (errcode_for_file_access(), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2130671d36..b6cb4111b8 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4262,7 +4262,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, { char *errormsg; - record = XLogReadRecord(xlogreader, RecPtr, &errormsg); + XLogReadRecord(xlogreader, RecPtr, &record, &errormsg); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index f8375eceeb..7d9438f0ea 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -264,20 +264,21 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) * If RecPtr is valid, try to read a record at that position. Otherwise * try to read a record just after the last one previously read. * - * If the read_page callback fails to read the requested data, NULL is - * returned. The callback is expected to have reported the error; errormsg - * is set to NULL. + * If the read_page callback fails to read the requested data, *record is set + * to NULL and XLREAD_FAIL is returned. The callback is expected to have + * reported the error; errormsg is set to NULL. * - * If the reading fails for some other reason, NULL is also returned, and - * *errormsg is set to a string with details of the failure. + * If the reading fails for some other reason, *record is also set to NULL and + * XLREAD_FAIL is returned. *errormsg is set to a string with details of the + * failure. * * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. */ -XLogRecord * -XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) +XLogReadRecordResult +XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogRecord **record, char **errormsg) { - XLogRecord *record; XLogRecPtr targetPagePtr; bool randAccess; uint32 len, @@ -384,8 +385,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) * cannot access any other fields until we've verified that we got the * whole header. */ - record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); - total_len = record->xl_tot_len; + *record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); + total_len = (*record)->xl_tot_len; /* * If the whole record header is on this page, validate it immediately. @@ -397,7 +398,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) */ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) { - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record, + if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, *record, randAccess)) goto err; gotheader = true; @@ -519,9 +520,9 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* If we just reassembled the record header, validate it. */ if (!gotheader) { - record = (XLogRecord *) state->readRecordBuf; + *record = (XLogRecord *) state->readRecordBuf; if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, - record, randAccess)) + *record, randAccess)) goto err; gotheader = true; } @@ -529,8 +530,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) Assert(gotheader); - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecord(state, record, RecPtr)) + *record = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecord(state, *record, RecPtr)) goto err; pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); @@ -551,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) goto err; /* Record does not cross a page boundary */ - if (!ValidXLogRecord(state, record, RecPtr)) + if (!ValidXLogRecord(state, *record, RecPtr)) goto err; state->EndRecPtr = RecPtr + MAXALIGN(total_len); @@ -562,18 +563,19 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* * Special processing if it's an XLOG SWITCH record */ - if (record->xl_rmid == RM_XLOG_ID && - (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) + if ((*record)->xl_rmid == RM_XLOG_ID && + ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) { /* Pretend it extends to end of segment */ state->EndRecPtr += state->wal_segment_size - 1; state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size); } - if (DecodeXLogRecord(state, record, errormsg)) - return record; - else - return NULL; + if (DecodeXLogRecord(state, *record, errormsg)) + return XLREAD_SUCCESS; + + *record = NULL; + return XLREAD_FAIL; err: @@ -586,7 +588,8 @@ err: if (state->errormsg_buf[0] != '\0') *errormsg = state->errormsg_buf; - return NULL; + *record = NULL; + return XLREAD_FAIL; } /* @@ -990,6 +993,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr tmpRecPtr; XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; + XLogRecord *record; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); @@ -1078,7 +1082,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * because either we're at the first record after the beginning of a page * or we just jumped over the remaining data of a continuation. */ - while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL) + while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS) { /* continue after the record */ tmpRecPtr = InvalidXLogRecPtr; diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index acb4d9a106..d86feb0a0e 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -481,7 +481,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - record = XLogReadRecord(ctx->reader, startptr, &err); + XLogReadRecord(ctx->reader, startptr, &record, &err); if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 16c6095179..6fc78d7445 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -289,7 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - record = XLogReadRecord(ctx->reader, startptr, &errm); + XLogReadRecord(ctx->reader, startptr, &record, &errm); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 182fe5bc82..d261b402eb 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -433,7 +433,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - record = XLogReadRecord(ctx->reader, startlsn, &errm); + XLogReadRecord(ctx->reader, startlsn, &record, &errm); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 1ef952b39a..7ca1536eaf 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2821,7 +2821,7 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm); + XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm); logical_startptr = InvalidXLogRecPtr; /* xlog record was invalid */ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index c3b9e738f7..b110559e63 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -76,7 +76,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, do { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + XLogReadRecord(xlogreader, startpoint, &record, &errormsg); if (record == NULL) { @@ -128,7 +128,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) if (xlogreader == NULL) pg_fatal("out of memory"); - record = XLogReadRecord(xlogreader, ptr, &errormsg); + XLogReadRecord(xlogreader, ptr, &record, &errormsg); if (record == NULL) { if (errormsg) @@ -191,7 +191,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, { uint8 info; - record = XLogReadRecord(xlogreader, searchptr, &errormsg); + XLogReadRecord(xlogreader, searchptr, &record, &errormsg); if (record == NULL) { diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 9ad70a2f5c..280e4754ca 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1125,7 +1125,7 @@ main(int argc, char **argv) for (;;) { /* try to read the next record */ - record = XLogReadRecord(xlogreader_state, first_record, &errormsg); + XLogReadRecord(xlogreader_state, first_record, &record, &errormsg); if (!record) { if (!config.follow || private.endptr_reached) diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 268252e88f..c8bd7afebe 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -66,6 +66,13 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* Return code from XLogReadRecord */ +typedef enum XLogReadRecordResult +{ + XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_FAIL /* failed during reading a record */ +} XLogReadRecordResult; + /* internal state of XLogNeedData() */ typedef enum xlnd_stateid { @@ -227,8 +234,8 @@ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, extern void XLogReaderFree(XLogReaderState *state); /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ -extern struct XLogRecord *XLogReadRecord(XLogReaderState *state, - XLogRecPtr recptr, char **errormsg); +extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state, + XLogRecPtr recptr, XLogRecord **record, char **errormsg); /* Validate a page */ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, -- 2.16.3 From d0993b0c0ea3a1a4e65ed2b83a51d192f81b7c39 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 09:55:58 +0900 Subject: [PATCH 04/10] Make XLogReadRecord a state machine This patch moves the caller sites of the callback above XLogReadRecord by making XLogReadRecord a state machine. --- src/backend/access/transam/twophase.c | 8 +- src/backend/access/transam/xlog.c | 10 ++- src/backend/access/transam/xlogreader.c | 109 ++++++++++++++++--------- src/backend/replication/logical/logical.c | 10 ++- src/backend/replication/logical/logicalfuncs.c | 10 ++- src/backend/replication/slotfuncs.c | 10 ++- src/backend/replication/walsender.c | 10 ++- src/bin/pg_rewind/parsexlog.c | 28 ++++++- src/bin/pg_waldump/pg_waldump.c | 11 ++- src/include/access/xlogreader.h | 20 ++++- 10 files changed, 175 insertions(+), 51 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 5dba27e5dd..a3573ad0af 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1394,7 +1394,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errmsg("out of memory"), errdetail("Failed while allocating a WAL reading processor."))); - XLogReadRecord(xlogreader, lsn, &record, &errormsg); + while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, xlogreader->loadLen, + xlogreader->currRecPtr, xlogreader->readBuf, + &xlogreader->readPageTLI); + if (record == NULL) ereport(ERROR, (errcode_for_file_access(), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b6cb4111b8..59fd12153a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4258,11 +4258,19 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, /* This is the first attempt to read this page. */ lastSourceFailed = false; + XLREAD_RESET(xlogreader); + for (;;) { char *errormsg; - XLogReadRecord(xlogreader, RecPtr, &record, &errormsg); + while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) + == XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, xlogreader->loadLen, + xlogreader->currRecPtr, xlogreader->readBuf, + &xlogreader->readPageTLI); + ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 7d9438f0ea..05a57a1ebd 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -261,31 +261,56 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) /* * Attempt to read an XLOG record. * - * If RecPtr is valid, try to read a record at that position. Otherwise - * try to read a record just after the last one previously read. + * This function runs a state machine and may need to call several times until + * a record is read. * - * If the read_page callback fails to read the requested data, *record is set - * to NULL and XLREAD_FAIL is returned. The callback is expected to have - * reported the error; errormsg is set to NULL. + * At the initial state, if called with valid pRecPtr, try to read a + * record at that position. Otherwise try to read a record just after + * the last one previously read. * - * If the reading fails for some other reason, *record is also set to NULL and - * XLREAD_FAIL is returned. *errormsg is set to a string with details of the - * failure. + * When a record is successfully read, returns XLREAD_SUCCESS and the + * result record is stored in *record then reset to initial state. + * + * Returns XLREAD_NEED_DATA if more data is needed. The caller shall + * read some more XLOG data into readBuf and call this function again. + * In that case loadPagePtr and loadLen in state is set to inform the + * required WAL data. The caller shall read in the requested data into + * readBuf and set readLen and readPageTLI to the length of the data + * actually read and the TLI for the data read in respectively. In + * case of failure readLen shall be set to -1 to inform error and + * store error message in errormsg_buf. + * + * If the reading fails for some other reason, *record is also set to + * NULL and XLREAD_FAIL is returned. *errormsg is set to a string with + * details of the failure. Reset to initial state. * * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. */ XLogReadRecordResult -XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, +XLogReadRecord(XLogReaderState *state, XLogRecPtr pRecPtr, XLogRecord **record, char **errormsg) { - XLogRecPtr targetPagePtr; - bool randAccess; - uint32 len, - total_len; - uint32 targetRecOff; - uint32 pageHeaderSize; - bool gotheader; + /* + * This function is a state machine that can exit and reenter at any place + * marked as XLR_LEAVE. All internal state need to be preserved through + * multiple calls. + */ + static XLogRecPtr targetPagePtr; + static bool randAccess; + static uint32 len, + total_len; + static uint32 targetRecOff; + static uint32 pageHeaderSize; + static bool gotheader; + static XLogRecPtr RecPtr; + +#define XLR_STATE state->xlread_state +#define XLR_INIT_STATE XLREAD_STATE_INIT + + XLR_SWITCH(); + + RecPtr = pRecPtr; /* * randAccess indicates whether to verify the previous-record pointer of @@ -339,10 +364,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, */ while (XLogNeedData(state, targetPagePtr, Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); - + XLR_LEAVE(XLREAD_STATE_READ_PAGE, XLREAD_NEED_DATA); + if (state->readLen < 0) goto err; @@ -421,10 +444,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (total_len > len) { /* Need to reassemble record */ - char *contdata; - XLogPageHeader pageHeader; - char *buffer; - uint32 gotlen; + static char *contdata; + static XLogPageHeader pageHeader; + static char *buffer; + static uint32 gotlen; /* * Enlarge readRecordBuf as needed. @@ -454,9 +477,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, while (XLogNeedData(state, targetPagePtr, Min(total_len - gotlen + SizeOfXLogShortPHD, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_NEXT_PAGE, XLREAD_NEED_DATA); if (state->readLen < 0) goto err; @@ -493,9 +514,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (state->readLen < pageHeaderSize) { while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_PAGE_HADER, XLREAD_NEED_DATA); } Assert(pageHeaderSize <= state->readLen); @@ -508,9 +527,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (state->readLen < pageHeaderSize + len) { if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_CONTRECORD, XLREAD_NEED_DATA); } memcpy(buffer, (char *) contdata, len); @@ -544,9 +561,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, /* Wait for the record data to become available */ while (XLogNeedData(state, targetPagePtr, Min(targetRecOff + total_len, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_RECORD, XLREAD_NEED_DATA); if (state->readLen < 0) goto err; @@ -560,6 +575,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, state->ReadRecPtr = RecPtr; } + XLR_SWITCH_END(); + /* * Special processing if it's an XLOG SWITCH record */ @@ -572,10 +589,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, } if (DecodeXLogRecord(state, *record, errormsg)) - return XLREAD_SUCCESS; + XLR_RETURN(XLREAD_SUCCESS); *record = NULL; - return XLREAD_FAIL; + XLR_RETURN(XLREAD_FAIL); err: @@ -589,7 +606,7 @@ err: *errormsg = state->errormsg_buf; *record = NULL; - return XLREAD_FAIL; + XLR_RETURN(XLREAD_FAIL); } /* @@ -615,6 +632,8 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) static XLogSegNo targetSegNo; static XLogPageHeader hdr; +#undef XLR_STATE +#undef XLR_INIT_STATE #define XLR_STATE state->xlnd_state #define XLR_INIT_STATE XLND_STATE_INIT @@ -994,6 +1013,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; XLogRecord *record; + XLogReadRecordResult result; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); @@ -1082,8 +1102,17 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * because either we're at the first record after the beginning of a page * or we just jumped over the remaining data of a continuation. */ - while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS) + while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) != + XLREAD_FAIL) { + if (result == XLREAD_NEED_DATA) + { + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + continue; + } + /* continue after the record */ tmpRecPtr = InvalidXLogRecPtr; diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index d86feb0a0e..1740753b76 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -481,7 +481,15 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - XLogReadRecord(ctx->reader, startptr, &record, &err); + while (XLogReadRecord(ctx->reader, startptr, &record, &err) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 6fc78d7445..4d09255504 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -289,7 +289,15 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - XLogReadRecord(ctx->reader, startptr, &record, &errm); + while (XLogReadRecord(ctx->reader, startptr, &record, &errm) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index d261b402eb..4a8952931d 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -433,7 +433,15 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - XLogReadRecord(ctx->reader, startlsn, &record, &errm); + while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7ca1536eaf..cb85ba3abf 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2821,7 +2821,15 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm); + while (XLogReadRecord(logical_decoding_ctx->reader, + logical_startptr, &record, &errm) == XLREAD_NEED_DATA) + logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader, + logical_decoding_ctx->reader->loadPagePtr, + logical_decoding_ctx->reader->loadLen, + logical_decoding_ctx->reader->currRecPtr, + logical_decoding_ctx->reader->readBuf, + &logical_decoding_ctx->reader->readPageTLI); + logical_startptr = InvalidXLogRecPtr; /* xlog record was invalid */ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index b110559e63..21b638ed34 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -76,7 +76,14 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, do { - XLogReadRecord(xlogreader, startpoint, &record, &errormsg); + while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); if (record == NULL) { @@ -128,7 +135,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) if (xlogreader == NULL) pg_fatal("out of memory"); - XLogReadRecord(xlogreader, ptr, &record, &errormsg); + while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); + if (record == NULL) { if (errormsg) @@ -191,7 +206,14 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, { uint8 info; - XLogReadRecord(xlogreader, searchptr, &record, &errormsg); + while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); if (record == NULL) { diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 280e4754ca..acee7ae199 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1125,7 +1125,16 @@ main(int argc, char **argv) for (;;) { /* try to read the next record */ - XLogReadRecord(xlogreader_state, first_record, &record, &errormsg); + while (XLogReadRecord(xlogreader_state, + first_record, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader_state->read_page(xlogreader_state, + xlogreader_state->loadPagePtr, + xlogreader_state->loadLen, + xlogreader_state->currRecPtr, + xlogreader_state->readBuf, + &xlogreader_state->readPageTLI); + if (!record) { if (!config.follow || private.endptr_reached) diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index c8bd7afebe..338dc2c14d 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -70,6 +70,7 @@ typedef struct typedef enum XLogReadRecordResult { XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */ XLREAD_FAIL /* failed during reading a record */ } XLogReadRecordResult; @@ -82,6 +83,17 @@ typedef enum xlnd_stateid XLND_STATE_PAGEFULLHEAD } xlnd_stateid; +/* internal state of XLogReadRecord() */ +typedef enum xlread_stateid +{ + XLREAD_STATE_INIT, + XLREAD_STATE_READ_PAGE, + XLREAD_STATE_READ_NEXT_PAGE, + XLREAD_STATE_READ_PAGE_HADER, + XLREAD_STATE_READ_CONTRECORD, + XLREAD_STATE_READ_RECORD +} xlread_stateid; + struct XLogReaderState { /* ---------------------------------------- @@ -217,12 +229,18 @@ struct XLogReaderState /* Buffer to hold error message */ char *errormsg_buf; - /* Internal state of XLogNeedData */ + /* Internal state of XLogNeedData and XLogReadRecord */ union { xlnd_stateid c; void *j; } xlnd_state; + + union + { + xlread_stateid c; + void *j; + } xlread_state; }; /* Get a new XLogReader */ -- 2.16.3 From 39538903779a919cd76aba297b4f1e2350bb4a44 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 13:48:25 +0900 Subject: [PATCH 05/10] Make XLogPageRead not use callback but call the function directly This patch replaces the call to the callback in XLogReadRecrod with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/xlog.c | 44 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 59fd12153a..ba75ecb69c 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,13 +804,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */ static XLogSource currentSource = 0; /* XLOG_FROM_* code */ static bool lastSourceFailed = false; -typedef struct XLogPageReadPrivate -{ - int emode; - bool fetching_ckpt; /* are we fetching a checkpoint record? */ - bool randAccess; -} XLogPageReadPrivate; - /* * These variables track when we last obtained some WAL data to process, * and where we got it from. (XLogReceiptSource is initially the same as @@ -885,9 +878,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *readTLI); +static void XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, bool fetching_ckpt, XLogRecPtr tliRecPtr); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); @@ -4248,12 +4240,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, bool fetching_ckpt) { XLogRecord *record; - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - - /* Pass through parameters to XLogPageRead */ - private->fetching_ckpt = fetching_ckpt; - private->emode = emode; - private->randAccess = (RecPtr != InvalidXLogRecPtr); + bool randAccess = (RecPtr != InvalidXLogRecPtr); /* This is the first attempt to read this page. */ lastSourceFailed = false; @@ -4266,10 +4253,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, xlogreader->loadLen, - xlogreader->currRecPtr, xlogreader->readBuf, - &xlogreader->readPageTLI); + XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; @@ -6205,7 +6189,6 @@ StartupXLOG(void) bool backupFromStandby = false; DBState dbstate_at_startup; XLogReaderState *xlogreader; - XLogPageReadPrivate private; bool fast_promoted = false; struct stat st; @@ -6346,8 +6329,7 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - MemSet(&private, 0, sizeof(XLogPageReadPrivate)); - xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -11516,12 +11498,14 @@ CancelBackup(void) * sleep and retry. */ static void -XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI) +XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess) { - XLogPageReadPrivate *private = - (XLogPageReadPrivate *) xlogreader->private_data; - int emode = private->emode; + XLogRecPtr targetPagePtr = xlogreader->loadPagePtr; + int reqLen = xlogreader->loadLen; + XLogRecPtr targetRecPtr = xlogreader->currRecPtr; + char *readBuf = xlogreader->readBuf; + TimeLineID *readTLI = &xlogreader->readPageTLI; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11564,8 +11548,8 @@ retry: receivedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - private->randAccess, - private->fetching_ckpt, + randAccess, + fetching_ckpt, targetRecPtr)) { if (readFile >= 0) -- 2.16.3 From 5aa9744854c4ed044526ac752fa982d3af4e4f0a Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:02:19 +0900 Subject: [PATCH 06/10] Make XlogReadTwoPhaseData not use callback but call the function directly This patch replaces the call to the callback in XlogReadTwoPhaseData with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/twophase.c | 8 ++------ src/backend/access/transam/xlogutils.c | 8 +++++--- src/backend/replication/logical/logicalfuncs.c | 8 ++++++-- src/include/access/xlogutils.h | 5 +---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index a3573ad0af..caaa09d785 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1386,8 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, - NULL); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -1396,10 +1395,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, xlogreader->loadLen, - xlogreader->currRecPtr, xlogreader->readBuf, - &xlogreader->readPageTLI); + read_local_xlog_page(xlogreader); if (record == NULL) ereport(ERROR, diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index c853e1f0e3..fd461f16fc 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -908,10 +908,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * loop for now. */ void -read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page, - TimeLineID *pageTLI) +read_local_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *cur_page = state->readBuf; + TimeLineID *pageTLI = &state->readPageTLI; XLogRecPtr read_upto, loc; int count; diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 4d09255504..240a375d8f 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -118,8 +118,12 @@ void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { - read_local_xlog_page(state, targetPagePtr, reqLen, - targetRecPtr, cur_page, pageTLI); + Assert(targetPagePtr == state->loadPagePtr && + reqLen == state->loadLen && + targetRecPtr == state->currRecPtr && + cur_page == state->readBuf && + pageTLI == &state->readPageTLI); + read_local_xlog_page(state); } /* diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 5dba86b8b8..7f119837ce 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern void read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page, - TimeLineID *pageTLI); +extern void read_local_xlog_page(XLogReaderState *state); extern void XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength); -- 2.16.3 From 2ad57f89065d769ea84a543eefed06d18543010b Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:35:10 +0900 Subject: [PATCH 07/10] Make logical rep stuff not use callback but call the function directly This is a bit different from the two before. This patch moves the callback from XLogReaderState to LogicalDecodingContext. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/replication/logical/logical.c | 19 +++++++++---------- src/backend/replication/logical/logicalfuncs.c | 19 +++++-------------- src/backend/replication/slotfuncs.c | 9 +++------ src/backend/replication/walsender.c | 17 +++++++++-------- src/include/replication/logical.h | 13 +++++++++---- src/include/replication/logicalfuncs.h | 5 +---- 6 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 1740753b76..6dc87207b0 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -123,7 +123,7 @@ StartupDecodingContext(List *output_plugin_options, TransactionId xmin_horizon, bool need_full_snapshot, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -172,12 +172,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx); + ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->read_page = read_page; + ctx->reorder = ReorderBufferAllocate(); ctx->snapshot_builder = AllocateSnapshotBuilder(ctx->reorder, xmin_horizon, start_lsn, @@ -231,7 +233,7 @@ CreateInitDecodingContext(char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -373,7 +375,7 @@ LogicalDecodingContext * CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -474,6 +476,8 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) (uint32) (slot->data.restart_lsn >> 32), (uint32) slot->data.restart_lsn); + XLREAD_RESET(ctx->reader); + /* Wait for a consistent starting point */ for (;;) { @@ -483,12 +487,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) /* the read_page callback waits for new WAL */ while (XLogReadRecord(ctx->reader, startptr, &record, &err) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (err) elog(ERROR, "%s", err); diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 240a375d8f..0c4bc62cfa 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -115,15 +115,9 @@ check_permissions(void) } void -logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) +logical_read_local_xlog_page(LogicalDecodingContext *ctx) { - Assert(targetPagePtr == state->loadPagePtr && - reqLen == state->loadLen && - targetRecPtr == state->currRecPtr && - cur_page == state->readBuf && - pageTLI == &state->readPageTLI); - read_local_xlog_page(state); + read_local_xlog_page(ctx->reader); } /* @@ -286,6 +280,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin /* invalidate non-timetravel entries */ InvalidateSystemCaches(); + XLREAD_RESET(ctx->reader); + /* Decode until we run out of records */ while ((startptr != InvalidXLogRecPtr && startptr < end_of_wal) || (ctx->reader->EndRecPtr != InvalidXLogRecPtr && ctx->reader->EndRecPtr < end_of_wal)) @@ -295,12 +291,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin while (XLogReadRecord(ctx->reader, startptr, &record, &errm) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4a8952931d..afdd3dea37 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -420,6 +420,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) /* invalidate non-timetravel entries */ InvalidateSystemCaches(); + XLREAD_RESET(ctx->reader); + /* Decode at least one record, until we run out of records */ while ((!XLogRecPtrIsInvalid(startlsn) && startlsn < moveto) || @@ -435,12 +437,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) */ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index cb85ba3abf..5ac3ef4c66 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -762,9 +762,13 @@ StartReplication(StartReplicationCmd *cmd) * set every time WAL is flushed. */ static void -logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) +logical_read_xlog_page(LogicalDecodingContext *ctx) { + XLogReaderState *state = ctx->reader; + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *cur_page = state->readBuf; + XLogRecPtr flushptr; int count; @@ -2821,14 +2825,11 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; + XLREAD_RESET(logical_decoding_ctx->reader); + while (XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm) == XLREAD_NEED_DATA) - logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader, - logical_decoding_ctx->reader->loadPagePtr, - logical_decoding_ctx->reader->loadLen, - logical_decoding_ctx->reader->currRecPtr, - logical_decoding_ctx->reader->readBuf, - &logical_decoding_ctx->reader->readPageTLI); + logical_decoding_ctx->read_page(logical_decoding_ctx); logical_startptr = InvalidXLogRecPtr; diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index 0a2a63a48c..70339eb8be 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -32,7 +32,11 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) ( TransactionId xid ); -typedef struct LogicalDecodingContext +typedef struct LogicalDecodingContext LogicalDecodingContext; + +typedef void (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx); + +struct LogicalDecodingContext { /* memory context this is all allocated in */ MemoryContext context; @@ -42,6 +46,7 @@ typedef struct LogicalDecodingContext /* infrastructure pieces for decoding */ XLogReaderState *reader; + LogicalDecodingXLogReadPageCB read_page; struct ReorderBuffer *reorder; struct SnapBuild *snapshot_builder; @@ -89,7 +94,7 @@ typedef struct LogicalDecodingContext bool prepared_write; XLogRecPtr write_location; TransactionId write_xid; -} LogicalDecodingContext; +}; extern void CheckLogicalDecodingRequirements(void); @@ -98,7 +103,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); @@ -106,7 +111,7 @@ extern LogicalDecodingContext *CreateDecodingContext( XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h index 3d00dee067..04a9fe10fa 100644 --- a/src/include/replication/logicalfuncs.h +++ b/src/include/replication/logicalfuncs.h @@ -11,9 +11,6 @@ #include "replication/logical.h" -extern void logical_read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, - char *cur_page, TimeLineID *pageTLI); +extern void logical_read_local_xlog_page(LogicalDecodingContext *ctx); #endif -- 2.16.3 From 8fcca6af6025eaca9e9857cdeaf8c4b39a851c34 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:50:51 +0900 Subject: [PATCH 08/10] Make pg_waldump not use callback but call the function directly This patch does the similar thing to the change in logical rep. Moves callback from XLogReaderState from the parameter of XLogFindNextRecord. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/xlogreader.c | 17 +++++++---------- src/bin/pg_waldump/pg_waldump.c | 21 +++++++++------------ src/include/access/xlogreader.h | 14 +++++--------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 05a57a1ebd..72b82a17d6 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1006,7 +1006,8 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * debugging purposes. */ XLogRecPtr -XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) +XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private) { XLogReaderState saved_state = *state; XLogRecPtr tmpRecPtr; @@ -1018,6 +1019,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) Assert(!XLogRecPtrIsInvalid(RecPtr)); + XLREAD_RESET(state); + /* * skip over potential continuation data, keeping in mind that it may span * multiple pages @@ -1045,9 +1048,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) /* Read the page containing the record */ while(XLogNeedData(state, targetPagePtr, targetRecOff)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); if (state->readLen < 0) goto err; @@ -1058,9 +1059,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) /* make sure we have enough data for the page header */ while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); if (state->readLen < 0) goto err; @@ -1107,9 +1106,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) { if (result == XLREAD_NEED_DATA) { - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); continue; } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index acee7ae199..1966da493f 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, * XLogReader read_page callback */ static void -XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI) +XLogDumpReadPage(XLogReaderState *state, void *priv) { - XLogDumpPrivate *private = state->private_data; + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *readBuff = state->readBuf; + XLogDumpPrivate *private = (XLogDumpPrivate *) priv; int count = XLOG_BLCKSZ; if (private->endptr != InvalidXLogRecPtr) @@ -1095,13 +1097,13 @@ main(int argc, char **argv) /* done with argument parsing, do the actual work */ /* we have everything we need, start reading */ - xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage, - &private); + xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL); if (!xlogreader_state) fatal_error("out of memory"); /* first find a valid recptr to start from */ - first_record = XLogFindNextRecord(xlogreader_state, private.startptr); + first_record = XLogFindNextRecord(xlogreader_state, private.startptr, + &XLogDumpReadPage, (void*) &private); if (first_record == InvalidXLogRecPtr) fatal_error("could not find a valid record after %X/%X", @@ -1128,12 +1130,7 @@ main(int argc, char **argv) while (XLogReadRecord(xlogreader_state, first_record, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader_state->read_page(xlogreader_state, - xlogreader_state->loadPagePtr, - xlogreader_state->loadLen, - xlogreader_state->currRecPtr, - xlogreader_state->readBuf, - &xlogreader_state->readPageTLI); + XLogDumpReadPage(xlogreader_state, (void *) &private); if (!record) { diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 338dc2c14d..5f85c79424 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -29,14 +29,6 @@ typedef struct XLogReaderState XLogReaderState; -/* Function type definition for the read_page callback */ -typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, - XLogRecPtr targetRecPtr, - char *readBuf, - TimeLineID *pageTLI); - typedef struct { /* Is this block ref in use? */ @@ -263,7 +255,11 @@ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, extern void XLogReaderInvalReadState(XLogReaderState *state); #ifdef FRONTEND -extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); +/* Function type definition for the read_page callback */ +typedef void (*XLogFindNextRecordCB) (XLogReaderState *xlogreader, + void *private); +extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private); #endif /* FRONTEND */ /* Functions for decoding an XLogRecord */ -- 2.16.3 From 882d4dca9935233cbe5ed0f79845d78d691e6bb8 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 16:00:57 +0900 Subject: [PATCH 09/10] Make pg_rewind not use callback but call the function directly This patch replaces the call to the callback in pg_rewind with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/bin/pg_rewind/parsexlog.c | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 21b638ed34..d504c3227a 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -49,9 +49,7 @@ typedef struct XLogPageReadPrivate } XLogPageReadPrivate; static void SimpleXLogPageRead(XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI); + XLogPageReadPrivate *private); /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline @@ -69,8 +67,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -78,12 +75,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, { while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, &private); if (record == NULL) { @@ -130,19 +122,13 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, &private); if (record == NULL) { @@ -196,8 +182,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -208,12 +193,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, &private); if (record == NULL) { @@ -260,11 +240,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, /* XLogreader callback function, to read a WAL page */ static void -SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI) +SimpleXLogPageRead(XLogReaderState *xlogreader, XLogPageReadPrivate *private) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogRecPtr targetPagePtr = xlogreader->loadPagePtr; + char *readBuf = xlogreader->readBuf; + TimeLineID *pageTLI = &xlogreader->readPageTLI; uint32 targetPageOff; XLogRecPtr targetSegEnd; XLogSegNo targetSegNo; -- 2.16.3 From ef5b69b3faf496353a031d24cb151b3ebc37c45c Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 16:15:18 +0900 Subject: [PATCH 10/10] Remove callback entry from XLogReaderState This is the third (3/2?) step. Remove no-longer-useful members read_page and private_data from XLogReaderState. Then change XLogReaderAllocate not to take the parameters. --- src/backend/access/transam/twophase.c | 2 +- src/backend/access/transam/xlog.c | 4 ++-- src/backend/access/transam/xlogreader.c | 9 ++------- src/backend/replication/logical/logical.c | 2 +- src/bin/pg_rewind/parsexlog.c | 6 +++--- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 32 +------------------------------ 7 files changed, 11 insertions(+), 46 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index caaa09d785..e70245e7cb 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1386,7 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + xlogreader = XLogReaderAllocate(wal_segment_size); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index ba75ecb69c..f34a9ba806 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1188,7 +1188,7 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + debug_reader = XLogReaderAllocate(wal_segment_size); if (!debug_reader) { @@ -6329,7 +6329,7 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + xlogreader = XLogReaderAllocate(wal_segment_size); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 72b82a17d6..e2d7b9060b 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -128,8 +128,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc, - void *private_data) +XLogReaderAllocate(int wal_segment_size) { XLogReaderState *state; @@ -157,11 +156,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc, } state->wal_segment_size = wal_segment_size; - state->read_page = pagereadfunc; - /* system_identifier initialized to zeroes above */ - state->private_data = private_data; - /* ReadRecPtr and EndRecPtr initialized to zeroes above */ - /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */ + /* All members are initialized to zeroes above */ state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1, MCXT_ALLOC_NO_OOM); if (!state->errormsg_buf) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 6dc87207b0..e257dd8afc 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -172,7 +172,7 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + ctx->reader = XLogReaderAllocate(wal_segment_size); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index d504c3227a..b46465fff3 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -67,7 +67,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -122,7 +122,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -182,7 +182,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 1966da493f..f70dc7bbc0 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1097,7 +1097,7 @@ main(int argc, char **argv) /* done with argument parsing, do the actual work */ /* we have everything we need, start reading */ - xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader_state = XLogReaderAllocate(WalSegSz); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 5f85c79424..f407746209 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -98,40 +98,12 @@ struct XLogReaderState */ int wal_segment_size; - /* - * Data input callback (mandatory). - * - * This callback shall read at least reqLen valid bytes of the xlog page - * starting at targetPagePtr, and store them in readBuf. The callback - * shall return the number of bytes read (never more than XLOG_BLCKSZ), or - * -1 on failure. The callback shall sleep, if necessary, to wait for the - * requested bytes to become available. The callback will not be invoked - * again for the same page unless more than the returned number of bytes - * are needed. - * - * targetRecPtr is the position of the WAL record we're reading. Usually - * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs - * to read and verify the page or segment header, before it reads the - * actual WAL record it's interested in. In that case, targetRecPtr can - * be used to determine which timeline to read the page from. - * - * The callback shall set *pageTLI to the TLI of the file the page was - * read from. It is currently used only for error reporting purposes, to - * reconstruct the name of the WAL file where an error occurred. - */ - XLogPageReadCB read_page; - /* * System identifier of the xlog files we're about to read. Set to zero * (the default value) if unknown or unimportant. */ uint64 system_identifier; - /* - * Opaque data for callbacks to use. Not used by XLogReader. - */ - void *private_data; - /* * Start and end point of last record read. EndRecPtr is also used as the * position to read next, if XLogReadRecord receives an invalid recptr. @@ -236,9 +208,7 @@ struct XLogReaderState }; /* Get a new XLogReader */ -extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - XLogPageReadCB pagereadfunc, - void *private_data); +extern XLogReaderState *XLogReaderAllocate(int wal_segment_size); /* Free an XLogReader */ extern void XLogReaderFree(XLogReaderState *state); -- 2.16.3
Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> wrote: > Hello. As mentioned before [1], read_page callback in > XLogReaderState is a cause of headaches. Adding another > remote-controlling stuff to xlog readers makes things messier [2]. The patch I posted in thread [2] tries to solve another problem: it tries to merge xlogutils.c:XLogRead(), walsender.c:XLogRead() and pg_waldump.c:XLogDumpXLogRead() into a single function, xlogutils.c:XLogRead(). > [2] > https://www.postgresql.org/message-id/20190412.122711.158276916.horiguchi.kyotaro@lab.ntt.co.jp > I refactored XLOG reading functions so that we don't need the > callback. I was curious about the patch, so I reviewed it: * xlogreader.c ** Comments mention "opcode", "op" and "expression step" - probably leftover from the executor, which seems to have inspired you. ** XLR_DISPATCH() seems to be unused ** Comment: "duplicatedly" -> "repeatedly" ? ** XLogReadRecord(): comment "All internal state need ..." -> "needs" ** XLogNeedData() *** shouldn't only the minimum amount of data needed (SizeOfXLogLongPHD) be requested here? state->loadLen = XLOG_BLCKSZ; XLR_LEAVE(XLND_STATE_SEGHEAD, true); Note that ->loadLen is also set only to the minimum amount of data needed elsewhere. *** you still mention "read_page callback" in a comment. *** state->readLen is checked before one of the calls of XLR_LEAVE(), but I think it should happen before *each* call. Otherwise data can be read from the page even if it's already in the buffer. * xlogreader.h ** XLND_STATE_PAGEFULLHEAD - maybe LONG rather than FULL? And perhaps HEAD -> HDR, so it's clear that it's about (page) header, not e.g. list head. ** XLogReaderState.loadLen - why not reqLen? loadLen sounds to me like "loaded" as opposed to "requested". And assignemnt like this int reqLen = xlogreader->loadLen; will also be less confusing with ->reqLen. Maybe also ->loadPagePtr should be renamed to ->targetPagePtr. * trailing whitespace: xlogreader.h:130, xlogreader.c:1058 * The 2nd argument of SimpleXLogPageRead() is "private", which seems too generic given that the function is no longer used as a callback. Since the XLogPageReadPrivate structure only has two fields, I think it'd be o.k. to pass them to the function directly. * I haven't found CF entry for this patch. -- Antonin Houska Web: https://www.cybertec-postgresql.com
Hello. Thank you for looking this. At Thu, 25 Apr 2019 13:58:20 +0200, Antonin Houska <ah@cybertec.at> wrote in <18581.1556193500@localhost> > Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> wrote: > > > Hello. As mentioned before [1], read_page callback in > > XLogReaderState is a cause of headaches. Adding another > > remote-controlling stuff to xlog readers makes things messier [2]. > > The patch I posted in thread [2] tries to solve another problem: it tries to > merge xlogutils.c:XLogRead(), walsender.c:XLogRead() and > pg_waldump.c:XLogDumpXLogRead() into a single function, > xlogutils.c:XLogRead(). > > > [2] > > https://www.postgresql.org/message-id/20190412.122711.158276916.horiguchi.kyotaro@lab.ntt.co.jp > > > I refactored XLOG reading functions so that we don't need the > > callback. > > I was curious about the patch, so I reviewed it: Thnak for the comment. (It's a shame that I might made it more complex..) > * xlogreader.c > > ** Comments mention "opcode", "op" and "expression step" - probably leftover > from the executor, which seems to have inspired you. Uggh. Yes, exactly. I believed change them all. Fixed. > ** XLR_DISPATCH() seems to be unused Right. XLR_ macros are used to dispatch internally in a function differently from EEO_ macros so I thought it uesless but I hesitated to remove it. I remove it. > ** Comment: "duplicatedly" -> "repeatedly" ? It aimed reentrance. But I notieced that it doesn't work when ERROR-exiting. So I remove the assertion and related code.. > ** XLogReadRecord(): comment "All internal state need ..." -> "needs" Fixed. > ** XLogNeedData() > > *** shouldn't only the minimum amount of data needed (SizeOfXLogLongPHD) > be requested here? > > state->loadLen = XLOG_BLCKSZ; > XLR_LEAVE(XLND_STATE_SEGHEAD, true); > > Note that ->loadLen is also set only to the minimum amount of data needed > elsewhere. Maybe right, but it is existing behavior so I preserved it as focusing on refactoring. > *** you still mention "read_page callback" in a comment. Thanks. "the read_page callback" were translated to "the caller" and it seems the last one. > *** state->readLen is checked before one of the calls of XLR_LEAVE(), but > I think it should happen before *each* call. Otherwise data can be read > from the page even if it's already in the buffer. That doesn't happen since XLogReadRecord doesn't LEAVE unless XLogNeedData returns true (that is, needs more data) and XLogNeedData returns true only when requested data is not on the buffer yet. (If I refactored it correctly and it seems to me so.) > * xlogreader.h > > ** XLND_STATE_PAGEFULLHEAD - maybe LONG rather than FULL? And perhaps HEAD > -> HDR, so it's clear that it's about (page) header, not e.g. list head. Perhaps that's better. Thanks. > > ** XLogReaderState.loadLen - why not reqLen? loadLen sounds to me like "loaded" > as opposed to "requested". And assignemnt like this > > int reqLen = xlogreader->loadLen; > > will also be less confusing with ->reqLen. > > Maybe also ->loadPagePtr should be renamed to ->targetPagePtr. Yeah, that's annoyance. reqLen *was* actually the "requested" length to XLogNeedData FKA ReadPageInternal, but in the current shape XLogNeedData makes different request to the callers (when fetching the first page in the newly visited segment), so the two (req(uest)Len and (to be)loadLen) are different things. At the same time, targetPagePoint is different from loadPagePtr. Of course the naming as arguable. > * trailing whitespace: xlogreader.h:130, xlogreader.c:1058 Thanks, it have been fixed on my repo. > * The 2nd argument of SimpleXLogPageRead() is "private", which seems too > generic given that the function is no longer used as a callback. Since the > XLogPageReadPrivate structure only has two fields, I think it'd be o.k. to > pass them to the function directly. Sound reasonable. Fixed. > * I haven't found CF entry for this patch. Yeah, I'll register this, maybe the week after next week. regards. -- Kyotaro Horiguchi NTT Open Source Software Center From cff0f38cc03c2186505cad30b27115bc0228b42f Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 10:22:49 +0900 Subject: [PATCH 01/10] Define macros to make XLogReadRecord a state machine To minimize apparent imapct on code, use some macros as syntax sugar. This is a similar stuff with ExecInterpExpr but a bitdifferent. The most significant difference is that this stuff allows some functions are leaved midst of their work thencontinue. Roughly speaking this is used as the follows. enum retval some_func() { static .. internal_variables; XLR_SWITCH(); ... XLR_LEAVE(STATUS1, RETVAL_CONTINUE); ... XLR_LEAVE(STATUS2, RETVAL_CONTINUE2); ... XLR_SWITCH_END(); XLR_RETURN(RETVAL_FINISH); } The caller uses the function as follows: while (some_func() != RETVAL_FINISH) { <do some work>; } --- src/backend/access/transam/xlogreader.c | 53 +++++++++++++++++++++++++++++++++ src/include/access/xlogreader.h | 3 ++ 2 files changed, 56 insertions(+) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 9196aa3aae..0e49ea6ab7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -29,6 +29,59 @@ #include "utils/memutils.h" #endif +/* + * Use computed-goto-based state dispatch when computed gotos are available. + * But use a separate symbol so that it's easy to adjust locally in this file + * for development and testing. + */ +#ifdef HAVE_COMPUTED_GOTO +#define XLR_USE_COMPUTED_GOTO +#endif /* HAVE_COMPUTED_GOTO */ + +/* + * Macros for state dispatch. + * + * XLR_SWITCH - just hides the switch if not in use. + * XLR_CASE - labels the implementation of named state. + * XLR_LEAVE - leave the function and return here at the next call. + * XLR_RETURN - return from the function and set state to initial state. + * XLR_END - just hides the closing brace if not in use. + */ +#if defined(XLR_USE_COMPUTED_GOTO) +#define XLR_SWITCH() \ + do { \ + if ((XLR_STATE).j) \ + goto *((void *) (XLR_STATE).j); \ + XLR_CASE(XLR_INIT_STATE); \ + } while (0) +#define XLR_CASE(name) name: +#define XLR_LEAVE(name, code) do { \ + (XLR_STATE).j = (&&name); return (code); \ + XLR_CASE(name); \ + } while (0) +#define XLR_RETURN(code) \ + do { \ + (XLR_STATE).j = (&&XLR_INIT_STATE); return (code); \ + } while (0) +#define XLR_SWITCH_END() +#else /* !XLR_USE_COMPUTED_GOTO */ +#define XLR_SWITCH() \ + /* Don't call duplicatedly */ \ + switch ((XLR_STATE).c) { \ + XLR_CASE(XLR_INIT_STATE); \ +#define XLR_CASE(name) case name: +#define XLR_LEAVE(name, code) \ + do { \ + (XLR_STATE).c = (name); return (code); \ + XLR_CASE(name); \ + } while (0) +#define XLR_RETURN(code) \ + do { \ + (XLR_STATE).c = (XLR_INIT_STATE); return (code); \ + } while (0) +#define XLR_SWITCH_END() } +#endif /* XLR_USE_COMPUTED_GOTO */ + static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f3bae0bf49..30500c35c7 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -240,6 +240,9 @@ extern bool DecodeXLogRecord(XLogReaderState *state, XLogRecord *record, #define XLogRecBlockImageApply(decoder, block_id) \ ((decoder)->blocks[block_id].apply_image) +/* Reset the reader state */ +#define XLREAD_RESET(state) ((state)->xlnd_state.j = (state)->xlread_state.j = 0) + extern bool RestoreBlockImage(XLogReaderState *recoder, uint8 block_id, char *dst); extern char *XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len); extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, -- 2.16.3 From d85e8b0dd061b8a7a8818586421720786b39845a Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 14:15:16 +0900 Subject: [PATCH 02/10] Make ReadPageInternal a state machine This patch set aims to remove read_page call back from XLogReaderState. This is done in two steps. This patch does the first stap. Makes ReadPageInternal, which currently calling read_page callback, a state machine and move the caller sites to XLogReadRecord and other direct callers of the callback. --- src/backend/access/transam/xlog.c | 15 ++- src/backend/access/transam/xlogreader.c | 179 +++++++++++++++---------- src/backend/access/transam/xlogutils.c | 8 +- src/backend/replication/logical/logicalfuncs.c | 6 +- src/backend/replication/walsender.c | 10 +- src/bin/pg_rewind/parsexlog.c | 17 ++- src/bin/pg_waldump/pg_waldump.c | 8 +- src/include/access/xlogreader.h | 46 +++++-- src/include/access/xlogutils.h | 2 +- src/include/replication/logicalfuncs.h | 2 +- 10 files changed, 183 insertions(+), 110 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c00b63c751..2130671d36 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -885,7 +885,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, +static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, @@ -11507,7 +11507,7 @@ CancelBackup(void) * XLogPageRead() to try fetching the record from another source, or to * sleep and retry. */ -static int +static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI) { @@ -11566,7 +11566,8 @@ retry: readLen = 0; readSource = 0; - return -1; + xlogreader->readLen = -1; + return; } } @@ -11661,7 +11662,8 @@ retry: goto next_record_is_invalid; } - return readLen; + xlogreader->readLen = readLen; + return; next_record_is_invalid: lastSourceFailed = true; @@ -11675,8 +11677,9 @@ next_record_is_invalid: /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - else - return -1; + + xlogreader->readLen = -1; + return; } /* diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0e49ea6ab7..311606c5cb 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -88,8 +88,7 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr); -static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, - int reqLen); +static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen); static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static void ResetDecoder(XLogReaderState *state); @@ -276,7 +275,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) uint32 targetRecOff; uint32 pageHeaderSize; bool gotheader; - int readOff; /* * randAccess indicates whether to verify the previous-record pointer of @@ -328,15 +326,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) * byte to cover the whole record header, or at least the part of it that * fits on the same page. */ - readOff = ReadPageInternal(state, - targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* - * ReadPageInternal always returns at least the page header, so we can - * examine it now. + * We have loaded at least the page header, so we can examine it now. */ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); if (targetRecOff == 0) @@ -362,8 +362,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) goto err; } - /* ReadPageInternal has verified the page header */ - Assert(pageHeaderSize <= readOff); + /* XLogNeedData has verified the page header */ + Assert(pageHeaderSize <= state->readLen); /* * Read the record length. @@ -440,14 +440,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) targetPagePtr += XLOG_BLCKSZ; /* Wait for the next page to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(total_len - gotlen + SizeOfXLogShortPHD, - XLOG_BLCKSZ)); + while (XLogNeedData(state, targetPagePtr, + Min(total_len - gotlen + SizeOfXLogShortPHD, + XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); - if (readOff < 0) + if (state->readLen < 0) goto err; - Assert(SizeOfXLogShortPHD <= readOff); + Assert(SizeOfXLogShortPHD <= state->readLen); /* Check that the continuation on next page looks valid */ pageHeader = (XLogPageHeader) state->readBuf; @@ -476,20 +479,28 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* Append the continuation from this page to the buffer */ pageHeaderSize = XLogPageHeaderSize(pageHeader); - if (readOff < pageHeaderSize) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize); + if (state->readLen < pageHeaderSize) + { + while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + } - Assert(pageHeaderSize <= readOff); + Assert(pageHeaderSize <= state->readLen); contdata = (char *) state->readBuf + pageHeaderSize; len = XLOG_BLCKSZ - pageHeaderSize; if (pageHeader->xlp_rem_len < len) len = pageHeader->xlp_rem_len; - if (readOff < pageHeaderSize + len) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize + len); + if (state->readLen < pageHeaderSize + len) + { + if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + } memcpy(buffer, (char *) contdata, len); buffer += len; @@ -520,9 +531,13 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) else { /* Wait for the record data to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + total_len, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + total_len, XLOG_BLCKSZ))) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* Record does not cross a page boundary */ @@ -565,39 +580,48 @@ err: } /* - * Read a single xlog page including at least [pageptr, reqLen] of valid data - * via the read_page() callback. + * Checks that an xlog page loaded in state->readBuf is including at least + * [pageptr, reqLen] and the page is valid. * - * Returns -1 if the required page cannot be read for some reason; errormsg_buf - * is set in that case (unless the error occurs in the read_page callback). + * Returns false if the required data is fully loaded. state->readLen is set to + * -1 when the loaded data is found to be invalid. * - * We fetch the page from a reader-local cache if we know we have the required - * data and if there hasn't been any error since caching the data. + * Otherwise, returns true and requests data using state->loadPagePtr and + * state->loadLen. The caller should load the region to state->readBuf and + * call this function again. */ -static int -ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) +static bool +XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) { - int readLen; - uint32 targetPageOff; - XLogSegNo targetSegNo; - XLogPageHeader hdr; + /* + * This function is a state machine that can exit and reenter at any place + * marked as XLR_LEAVE. All internal state is preserved through multiple + * calls. + */ + static uint32 targetPageOff; + static XLogSegNo targetSegNo; + static XLogPageHeader hdr; - Assert((pageptr % XLOG_BLCKSZ) == 0); +#define XLR_STATE state->xlnd_state +#define XLR_INIT_STATE XLND_STATE_INIT + + XLR_SWITCH (); XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size); targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size); + Assert((pageptr % XLOG_BLCKSZ) == 0); /* check whether we have all the requested data already */ if (targetSegNo == state->readSegNo && targetPageOff == state->readOff && reqLen <= state->readLen) - return state->readLen; + XLR_RETURN(false); /* * Data is not in our buffer. * * Every time we actually read the page, even if we looked at parts of it - * before, we need to do verification as the read_page callback might now - * be rereading data from a different source. + * before, we need to do verification as the caller might now be rereading + * data from a different source. * * Whenever switching to a new WAL segment, we read the first page of the * file and validate its header, even if that's not where the target @@ -606,18 +630,17 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) */ if (targetSegNo != state->readSegNo && targetPageOff != 0) { - XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; + state->loadPagePtr = pageptr - targetPageOff; + state->loadLen = XLOG_BLCKSZ; + XLR_LEAVE(XLND_STATE_SEGHEAD, true); - readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ, - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + if (state->readLen < 0) goto err; /* we can be sure to have enough WAL available, we scrolled back */ - Assert(readLen == XLOG_BLCKSZ); + Assert(state->readLen == XLOG_BLCKSZ); - if (!XLogReaderValidatePageHeader(state, targetSegmentPtr, + if (!XLogReaderValidatePageHeader(state, state->loadPagePtr, state->readBuf)) goto err; } @@ -626,48 +649,53 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) * First, read the requested data length, but at least a short page header * so that we can validate it. */ - readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + state->loadPagePtr = pageptr; + state->loadLen = Max(reqLen, SizeOfXLogShortPHD); + XLR_LEAVE(XLND_STATE_PAGEHEAD, true); + + if (state->readLen < 0) goto err; - Assert(readLen <= XLOG_BLCKSZ); + Assert(state->readLen <= XLOG_BLCKSZ); /* Do we have enough data to check the header length? */ - if (readLen <= SizeOfXLogShortPHD) + if (state->readLen <= SizeOfXLogShortPHD) goto err; - Assert(readLen >= reqLen); + Assert(state->readLen >= state->loadLen); hdr = (XLogPageHeader) state->readBuf; /* still not enough */ - if (readLen < XLogPageHeaderSize(hdr)) + if (state->readLen < XLogPageHeaderSize(hdr)) { - readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr), - state->currRecPtr, - state->readBuf, &state->readPageTLI); - if (readLen < 0) + state->loadPagePtr = pageptr; + state->loadLen = XLogPageHeaderSize(hdr); + XLR_LEAVE(XLND_STATE_PAGELONGHEAD, true); + + if (state->readLen < 0) goto err; } + XLR_SWITCH_END(); + /* * Now that we know we have the full header, validate it. */ - if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr)) - goto err; + if (!XLogReaderValidatePageHeader(state, pageptr, (char *) state->readBuf)) + goto err; /* update read state information */ state->readSegNo = targetSegNo; state->readOff = targetPageOff; - state->readLen = readLen; - return readLen; + XLR_RETURN(false); err: XLogReaderInvalReadState(state); - return -1; + state->readLen = -1; + + XLR_RETURN(false); } /* @@ -966,7 +994,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr targetPagePtr; int targetRecOff; uint32 pageHeaderSize; - int readLen; /* * Compute targetRecOff. It should typically be equal or greater than @@ -974,7 +1001,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * that, except when caller has explicitly specified the offset that * falls somewhere there or when we are skipping multi-page * continuation record. It doesn't matter though because - * ReadPageInternal() is prepared to handle that and will read at + * CheckPage() is prepared to handle that and will read at * least short page-header worth of data */ targetRecOff = tmpRecPtr % XLOG_BLCKSZ; @@ -983,8 +1010,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) targetPagePtr = tmpRecPtr - targetRecOff; /* Read the page containing the record */ - readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); - if (readLen < 0) + while(XLogNeedData(state, targetPagePtr, targetRecOff)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; header = (XLogPageHeader) state->readBuf; @@ -992,8 +1023,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) pageHeaderSize = XLogPageHeaderSize(header); /* make sure we have enough data for the page header */ - readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); - if (readLen < 0) + while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + + if (state->readLen < 0) goto err; /* skip over potential continuation data */ diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 10a663bae6..c853e1f0e3 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * exists for normal backends, so we have to do a check/sleep/repeat style of * loop for now. */ -int +void read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) @@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, else if (targetPagePtr + reqLen > read_upto) { /* not enough data there */ - return -1; + state->readLen = -1; + return; } else { @@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, XLOG_BLCKSZ); /* number of valid bytes in the buffer */ - return count; + state->readLen = count; + return; } diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index d974400d6e..16c6095179 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -114,12 +114,12 @@ check_permissions(void) (errmsg("must be superuser or replication role to use replication slots")))); } -int +void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { - return read_local_xlog_page(state, targetPagePtr, reqLen, - targetRecPtr, cur_page, pageTLI); + read_local_xlog_page(state, targetPagePtr, reqLen, + targetRecPtr, cur_page, pageTLI); } /* diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 440b6aac4b..1ef952b39a 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd) * which has to do a plain sleep/busy loop, because the walsender's latch gets * set every time WAL is flushed. */ -static int +static void logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { @@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* fail if not (implies we are going to shut down) */ if (flushptr < targetPagePtr + reqLen) - return -1; + { + state->readLen = -1; + return; + } if (targetPagePtr + XLOG_BLCKSZ <= flushptr) count = XLOG_BLCKSZ; /* more than one block available */ @@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* now actually read the data, we know it's there */ XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ); - return count; + state->readLen = count; + return; } /* diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 7709b96e00..c3b9e738f7 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -48,7 +48,7 @@ typedef struct XLogPageReadPrivate int tliIndex; } XLogPageReadPrivate; -static int SimpleXLogPageRead(XLogReaderState *xlogreader, +static void SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); @@ -237,7 +237,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, } /* XLogreader callback function, to read a WAL page */ -static int +static void SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) @@ -292,7 +292,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (xlogreadfd < 0) { pg_log_error("could not open file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return; } } @@ -305,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) { pg_log_error("could not seek in file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return; } @@ -318,13 +320,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, pg_log_error("could not read file \"%s\": read %d of %zu", xlogfpath, r, (Size) XLOG_BLCKSZ); - return -1; + xlogreader->readLen = -1; + return; } Assert(targetSegNo == xlogreadsegno); *pageTLI = targetHistory[private->tliIndex].tli; - return XLOG_BLCKSZ; + + xlogreader->readLen = XLOG_BLCKSZ; + return; } /* diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index e106fb2ed1..9ad70a2f5c 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, /* * XLogReader read_page callback */ -static int +static void XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI) { @@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, else { private->endptr_reached = true; - return -1; + state->readLen = -1; + return; } } XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr, readBuff, count); - return count; + state->readLen = count; + return; } /* diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 30500c35c7..f23d68c030 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -30,7 +30,7 @@ typedef struct XLogReaderState XLogReaderState; /* Function type definition for the read_page callback */ -typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader, +typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, @@ -66,6 +66,15 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* internal state of XLogNeedData() */ +typedef enum xlnd_stateid +{ + XLND_STATE_INIT, + XLND_STATE_SEGHEAD, + XLND_STATE_PAGEHEAD, + XLND_STATE_PAGELONGHEAD +} xlnd_stateid; + struct XLogReaderState { /* ---------------------------------------- @@ -120,6 +129,22 @@ struct XLogReaderState XLogRecPtr EndRecPtr; /* end+1 of last record read */ + /* ---------------------------------------- + * Communication with page reader + * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes. + * ---------------------------------------- + */ + /* parameters to page reader */ + XLogRecPtr loadPagePtr; /* Pointer to the page */ + int loadLen; /* wanted length in bytes */ + char *readBuf; /* buffer to store data */ + XLogRecPtr currRecPtr; /* beginning of the WAL record being read */ + + /* return from page reader */ + int32 readLen; /* bytes acutually read, must be larger than + * loadLen. -1 on error. */ + TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + /* ---------------------------------------- * Decoded representation of current record * @@ -145,17 +170,9 @@ struct XLogReaderState * ---------------------------------------- */ - /* - * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least - * readLen bytes) - */ - char *readBuf; - uint32 readLen; - - /* last read segment, segment offset, TLI for data currently in readBuf */ + /* last read segment and segment offset for data currently in readBuf */ XLogSegNo readSegNo; uint32 readOff; - TimeLineID readPageTLI; /* * beginning of prior page read, and its TLI. Doesn't necessarily @@ -164,8 +181,6 @@ struct XLogReaderState XLogRecPtr latestPagePtr; TimeLineID latestPageTLI; - /* beginning of the WAL record being read. */ - XLogRecPtr currRecPtr; /* timeline to read it from, 0 if a lookup is required */ TimeLineID currTLI; @@ -194,6 +209,13 @@ struct XLogReaderState /* Buffer to hold error message */ char *errormsg_buf; + + /* Internal state of XLogNeedData */ + union + { + xlnd_stateid c; + void *j; + } xlnd_state; }; /* Get a new XLogReader */ diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 0ab5ba62f5..5dba86b8b8 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern int read_local_xlog_page(XLogReaderState *state, +extern void read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI); diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h index 3fb7ad5d67..3d00dee067 100644 --- a/src/include/replication/logicalfuncs.h +++ b/src/include/replication/logicalfuncs.h @@ -11,7 +11,7 @@ #include "replication/logical.h" -extern int logical_read_local_xlog_page(XLogReaderState *state, +extern void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI); -- 2.16.3 From fa8ab3e2455ced1975199ba512f7a4e73b4bb23e Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 09:57:07 +0900 Subject: [PATCH 03/10] Change interface of XLogReadRecord As a preparation to the second step, this patch changes the interface of XLogReadRecord so that the function can request the callers for reading more data. --- src/backend/access/transam/twophase.c | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/access/transam/xlogreader.c | 52 ++++++++++++++------------ src/backend/replication/logical/logical.c | 2 +- src/backend/replication/logical/logicalfuncs.c | 2 +- src/backend/replication/slotfuncs.c | 2 +- src/backend/replication/walsender.c | 2 +- src/bin/pg_rewind/parsexlog.c | 6 +-- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 11 +++++- 10 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index a399c0052d..5dba27e5dd 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1394,7 +1394,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errmsg("out of memory"), errdetail("Failed while allocating a WAL reading processor."))); - record = XLogReadRecord(xlogreader, lsn, &errormsg); + XLogReadRecord(xlogreader, lsn, &record, &errormsg); if (record == NULL) ereport(ERROR, (errcode_for_file_access(), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2130671d36..b6cb4111b8 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4262,7 +4262,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, { char *errormsg; - record = XLogReadRecord(xlogreader, RecPtr, &errormsg); + XLogReadRecord(xlogreader, RecPtr, &record, &errormsg); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 311606c5cb..2c6109a254 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -254,20 +254,21 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) * If RecPtr is valid, try to read a record at that position. Otherwise * try to read a record just after the last one previously read. * - * If the read_page callback fails to read the requested data, NULL is - * returned. The callback is expected to have reported the error; errormsg - * is set to NULL. + * If the read_page callback fails to read the requested data, *record is set + * to NULL and XLREAD_FAIL is returned. The callback is expected to have + * reported the error; errormsg is set to NULL. * - * If the reading fails for some other reason, NULL is also returned, and - * *errormsg is set to a string with details of the failure. + * If the reading fails for some other reason, *record is also set to NULL and + * XLREAD_FAIL is returned. *errormsg is set to a string with details of the + * failure. * * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. */ -XLogRecord * -XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) +XLogReadRecordResult +XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogRecord **record, char **errormsg) { - XLogRecord *record; XLogRecPtr targetPagePtr; bool randAccess; uint32 len, @@ -374,8 +375,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) * cannot access any other fields until we've verified that we got the * whole header. */ - record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); - total_len = record->xl_tot_len; + *record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); + total_len = (*record)->xl_tot_len; /* * If the whole record header is on this page, validate it immediately. @@ -387,7 +388,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) */ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) { - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record, + if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, *record, randAccess)) goto err; gotheader = true; @@ -509,9 +510,9 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* If we just reassembled the record header, validate it. */ if (!gotheader) { - record = (XLogRecord *) state->readRecordBuf; + *record = (XLogRecord *) state->readRecordBuf; if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, - record, randAccess)) + *record, randAccess)) goto err; gotheader = true; } @@ -519,8 +520,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) Assert(gotheader); - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecord(state, record, RecPtr)) + *record = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecord(state, *record, RecPtr)) goto err; pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); @@ -541,7 +542,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) goto err; /* Record does not cross a page boundary */ - if (!ValidXLogRecord(state, record, RecPtr)) + if (!ValidXLogRecord(state, *record, RecPtr)) goto err; state->EndRecPtr = RecPtr + MAXALIGN(total_len); @@ -552,18 +553,19 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) /* * Special processing if it's an XLOG SWITCH record */ - if (record->xl_rmid == RM_XLOG_ID && - (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) + if ((*record)->xl_rmid == RM_XLOG_ID && + ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) { /* Pretend it extends to end of segment */ state->EndRecPtr += state->wal_segment_size - 1; state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size); } - if (DecodeXLogRecord(state, record, errormsg)) - return record; - else - return NULL; + if (DecodeXLogRecord(state, *record, errormsg)) + return XLREAD_SUCCESS; + + *record = NULL; + return XLREAD_FAIL; err: @@ -576,7 +578,8 @@ err: if (state->errormsg_buf[0] != '\0') *errormsg = state->errormsg_buf; - return NULL; + *record = NULL; + return XLREAD_FAIL; } /* @@ -980,6 +983,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr tmpRecPtr; XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; + XLogRecord *record; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); @@ -1068,7 +1072,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * because either we're at the first record after the beginning of a page * or we just jumped over the remaining data of a continuation. */ - while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL) + while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS) { /* continue after the record */ tmpRecPtr = InvalidXLogRecPtr; diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index acb4d9a106..d86feb0a0e 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -481,7 +481,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - record = XLogReadRecord(ctx->reader, startptr, &err); + XLogReadRecord(ctx->reader, startptr, &record, &err); if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 16c6095179..6fc78d7445 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -289,7 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - record = XLogReadRecord(ctx->reader, startptr, &errm); + XLogReadRecord(ctx->reader, startptr, &record, &errm); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 182fe5bc82..d261b402eb 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -433,7 +433,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - record = XLogReadRecord(ctx->reader, startlsn, &errm); + XLogReadRecord(ctx->reader, startlsn, &record, &errm); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 1ef952b39a..7ca1536eaf 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2821,7 +2821,7 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm); + XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm); logical_startptr = InvalidXLogRecPtr; /* xlog record was invalid */ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index c3b9e738f7..b110559e63 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -76,7 +76,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, do { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + XLogReadRecord(xlogreader, startpoint, &record, &errormsg); if (record == NULL) { @@ -128,7 +128,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) if (xlogreader == NULL) pg_fatal("out of memory"); - record = XLogReadRecord(xlogreader, ptr, &errormsg); + XLogReadRecord(xlogreader, ptr, &record, &errormsg); if (record == NULL) { if (errormsg) @@ -191,7 +191,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, { uint8 info; - record = XLogReadRecord(xlogreader, searchptr, &errormsg); + XLogReadRecord(xlogreader, searchptr, &record, &errormsg); if (record == NULL) { diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 9ad70a2f5c..280e4754ca 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1125,7 +1125,7 @@ main(int argc, char **argv) for (;;) { /* try to read the next record */ - record = XLogReadRecord(xlogreader_state, first_record, &errormsg); + XLogReadRecord(xlogreader_state, first_record, &record, &errormsg); if (!record) { if (!config.follow || private.endptr_reached) diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f23d68c030..17ee30fa36 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -66,6 +66,13 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* Return code from XLogReadRecord */ +typedef enum XLogReadRecordResult +{ + XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_FAIL /* failed during reading a record */ +} XLogReadRecordResult; + /* internal state of XLogNeedData() */ typedef enum xlnd_stateid { @@ -227,8 +234,8 @@ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, extern void XLogReaderFree(XLogReaderState *state); /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ -extern struct XLogRecord *XLogReadRecord(XLogReaderState *state, - XLogRecPtr recptr, char **errormsg); +extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state, + XLogRecPtr recptr, XLogRecord **record, char **errormsg); /* Validate a page */ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, -- 2.16.3 From 370046e589304acada216778d7ef82372e6de840 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Wed, 17 Apr 2019 09:55:58 +0900 Subject: [PATCH 04/10] Make XLogReadRecord a state machine This patch moves the caller sites of the callback above XLogReadRecord by making XLogReadRecord a state machine. --- src/backend/access/transam/twophase.c | 8 +- src/backend/access/transam/xlog.c | 10 ++- src/backend/access/transam/xlogreader.c | 109 ++++++++++++++++--------- src/backend/replication/logical/logical.c | 10 ++- src/backend/replication/logical/logicalfuncs.c | 10 ++- src/backend/replication/slotfuncs.c | 10 ++- src/backend/replication/walsender.c | 10 ++- src/bin/pg_rewind/parsexlog.c | 28 ++++++- src/bin/pg_waldump/pg_waldump.c | 11 ++- src/include/access/xlogreader.h | 20 ++++- 10 files changed, 175 insertions(+), 51 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 5dba27e5dd..a3573ad0af 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1394,7 +1394,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errmsg("out of memory"), errdetail("Failed while allocating a WAL reading processor."))); - XLogReadRecord(xlogreader, lsn, &record, &errormsg); + while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, xlogreader->loadLen, + xlogreader->currRecPtr, xlogreader->readBuf, + &xlogreader->readPageTLI); + if (record == NULL) ereport(ERROR, (errcode_for_file_access(), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b6cb4111b8..59fd12153a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4258,11 +4258,19 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, /* This is the first attempt to read this page. */ lastSourceFailed = false; + XLREAD_RESET(xlogreader); + for (;;) { char *errormsg; - XLogReadRecord(xlogreader, RecPtr, &record, &errormsg); + while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) + == XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, xlogreader->loadLen, + xlogreader->currRecPtr, xlogreader->readBuf, + &xlogreader->readPageTLI); + ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 2c6109a254..f953924a72 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -251,31 +251,56 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) /* * Attempt to read an XLOG record. * - * If RecPtr is valid, try to read a record at that position. Otherwise - * try to read a record just after the last one previously read. + * This function runs a state machine and may need to call several times until + * a record is read. * - * If the read_page callback fails to read the requested data, *record is set - * to NULL and XLREAD_FAIL is returned. The callback is expected to have - * reported the error; errormsg is set to NULL. + * At the initial state, if called with valid pRecPtr, try to read a + * record at that position. Otherwise try to read a record just after + * the last one previously read. * - * If the reading fails for some other reason, *record is also set to NULL and - * XLREAD_FAIL is returned. *errormsg is set to a string with details of the - * failure. + * When a record is successfully read, returns XLREAD_SUCCESS and the + * result record is stored in *record then reset to initial state. + * + * Returns XLREAD_NEED_DATA if more data is needed. The caller shall + * read some more XLOG data into readBuf and call this function again. + * In that case loadPagePtr and loadLen in state is set to inform the + * required WAL data. The caller shall read in the requested data into + * readBuf and set readLen and readPageTLI to the length of the data + * actually read and the TLI for the data read in respectively. In + * case of failure readLen shall be set to -1 to inform error and + * store error message in errormsg_buf. + * + * If the reading fails for some other reason, *record is also set to + * NULL and XLREAD_FAIL is returned. *errormsg is set to a string with + * details of the failure. Reset to initial state. * * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. */ XLogReadRecordResult -XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, +XLogReadRecord(XLogReaderState *state, XLogRecPtr pRecPtr, XLogRecord **record, char **errormsg) { - XLogRecPtr targetPagePtr; - bool randAccess; - uint32 len, - total_len; - uint32 targetRecOff; - uint32 pageHeaderSize; - bool gotheader; + /* + * This function is a state machine that can exit and reenter at any place + * marked as XLR_LEAVE. All internal state needs to be preserved through + * multiple calls. + */ + static XLogRecPtr targetPagePtr; + static bool randAccess; + static uint32 len, + total_len; + static uint32 targetRecOff; + static uint32 pageHeaderSize; + static bool gotheader; + static XLogRecPtr RecPtr; + +#define XLR_STATE state->xlread_state +#define XLR_INIT_STATE XLREAD_STATE_INIT + + XLR_SWITCH(); + + RecPtr = pRecPtr; /* * randAccess indicates whether to verify the previous-record pointer of @@ -329,10 +354,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, */ while (XLogNeedData(state, targetPagePtr, Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); - + XLR_LEAVE(XLREAD_STATE_READ_PAGE, XLREAD_NEED_DATA); + if (state->readLen < 0) goto err; @@ -411,10 +434,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (total_len > len) { /* Need to reassemble record */ - char *contdata; - XLogPageHeader pageHeader; - char *buffer; - uint32 gotlen; + static char *contdata; + static XLogPageHeader pageHeader; + static char *buffer; + static uint32 gotlen; /* * Enlarge readRecordBuf as needed. @@ -444,9 +467,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, while (XLogNeedData(state, targetPagePtr, Min(total_len - gotlen + SizeOfXLogShortPHD, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_NEXT_PAGE, XLREAD_NEED_DATA); if (state->readLen < 0) goto err; @@ -483,9 +504,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (state->readLen < pageHeaderSize) { while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_PAGE_HADER, XLREAD_NEED_DATA); } Assert(pageHeaderSize <= state->readLen); @@ -498,9 +517,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, if (state->readLen < pageHeaderSize + len) { if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_CONTRECORD, XLREAD_NEED_DATA); } memcpy(buffer, (char *) contdata, len); @@ -534,9 +551,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, /* Wait for the record data to become available */ while (XLogNeedData(state, targetPagePtr, Min(targetRecOff + total_len, XLOG_BLCKSZ))) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + XLR_LEAVE(XLREAD_STATE_READ_RECORD, XLREAD_NEED_DATA); if (state->readLen < 0) goto err; @@ -550,6 +565,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, state->ReadRecPtr = RecPtr; } + XLR_SWITCH_END(); + /* * Special processing if it's an XLOG SWITCH record */ @@ -562,10 +579,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, } if (DecodeXLogRecord(state, *record, errormsg)) - return XLREAD_SUCCESS; + XLR_RETURN(XLREAD_SUCCESS); *record = NULL; - return XLREAD_FAIL; + XLR_RETURN(XLREAD_FAIL); err: @@ -579,7 +596,7 @@ err: *errormsg = state->errormsg_buf; *record = NULL; - return XLREAD_FAIL; + XLR_RETURN(XLREAD_FAIL); } /* @@ -605,6 +622,8 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) static XLogSegNo targetSegNo; static XLogPageHeader hdr; +#undef XLR_STATE +#undef XLR_INIT_STATE #define XLR_STATE state->xlnd_state #define XLR_INIT_STATE XLND_STATE_INIT @@ -984,6 +1003,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; XLogRecord *record; + XLogReadRecordResult result; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); @@ -1072,8 +1092,17 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * because either we're at the first record after the beginning of a page * or we just jumped over the remaining data of a continuation. */ - while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS) + while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) != + XLREAD_FAIL) { + if (result == XLREAD_NEED_DATA) + { + state->read_page(state, state->loadPagePtr, state->loadLen, + state->currRecPtr, state->readBuf, + &state->readPageTLI); + continue; + } + /* continue after the record */ tmpRecPtr = InvalidXLogRecPtr; diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index d86feb0a0e..1740753b76 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -481,7 +481,15 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - XLogReadRecord(ctx->reader, startptr, &record, &err); + while (XLogReadRecord(ctx->reader, startptr, &record, &err) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 6fc78d7445..4d09255504 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -289,7 +289,15 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - XLogReadRecord(ctx->reader, startptr, &record, &errm); + while (XLogReadRecord(ctx->reader, startptr, &record, &errm) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index d261b402eb..4a8952931d 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -433,7 +433,15 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - XLogReadRecord(ctx->reader, startlsn, &record, &errm); + while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) == + XLREAD_NEED_DATA) + ctx->reader->read_page(ctx->reader, + ctx->reader->loadPagePtr, + ctx->reader->loadLen, + ctx->reader->currRecPtr, + ctx->reader->readBuf, + &ctx->reader->readPageTLI); + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7ca1536eaf..cb85ba3abf 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2821,7 +2821,15 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm); + while (XLogReadRecord(logical_decoding_ctx->reader, + logical_startptr, &record, &errm) == XLREAD_NEED_DATA) + logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader, + logical_decoding_ctx->reader->loadPagePtr, + logical_decoding_ctx->reader->loadLen, + logical_decoding_ctx->reader->currRecPtr, + logical_decoding_ctx->reader->readBuf, + &logical_decoding_ctx->reader->readPageTLI); + logical_startptr = InvalidXLogRecPtr; /* xlog record was invalid */ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index b110559e63..21b638ed34 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -76,7 +76,14 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, do { - XLogReadRecord(xlogreader, startpoint, &record, &errormsg); + while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); if (record == NULL) { @@ -128,7 +135,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) if (xlogreader == NULL) pg_fatal("out of memory"); - XLogReadRecord(xlogreader, ptr, &record, &errormsg); + while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); + if (record == NULL) { if (errormsg) @@ -191,7 +206,14 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, { uint8 info; - XLogReadRecord(xlogreader, searchptr, &record, &errormsg); + while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader->read_page(xlogreader, + xlogreader->loadPagePtr, + xlogreader->loadLen, + xlogreader->currRecPtr, + xlogreader->readBuf, + &xlogreader->readPageTLI); if (record == NULL) { diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 280e4754ca..acee7ae199 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1125,7 +1125,16 @@ main(int argc, char **argv) for (;;) { /* try to read the next record */ - XLogReadRecord(xlogreader_state, first_record, &record, &errormsg); + while (XLogReadRecord(xlogreader_state, + first_record, &record, &errormsg) == + XLREAD_NEED_DATA) + xlogreader_state->read_page(xlogreader_state, + xlogreader_state->loadPagePtr, + xlogreader_state->loadLen, + xlogreader_state->currRecPtr, + xlogreader_state->readBuf, + &xlogreader_state->readPageTLI); + if (!record) { if (!config.follow || private.endptr_reached) diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 17ee30fa36..9bfa9e8d54 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -70,6 +70,7 @@ typedef struct typedef enum XLogReadRecordResult { XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */ XLREAD_FAIL /* failed during reading a record */ } XLogReadRecordResult; @@ -82,6 +83,17 @@ typedef enum xlnd_stateid XLND_STATE_PAGELONGHEAD } xlnd_stateid; +/* internal state of XLogReadRecord() */ +typedef enum xlread_stateid +{ + XLREAD_STATE_INIT, + XLREAD_STATE_READ_PAGE, + XLREAD_STATE_READ_NEXT_PAGE, + XLREAD_STATE_READ_PAGE_HADER, + XLREAD_STATE_READ_CONTRECORD, + XLREAD_STATE_READ_RECORD +} xlread_stateid; + struct XLogReaderState { /* ---------------------------------------- @@ -217,12 +229,18 @@ struct XLogReaderState /* Buffer to hold error message */ char *errormsg_buf; - /* Internal state of XLogNeedData */ + /* Internal state of XLogNeedData and XLogReadRecord */ union { xlnd_stateid c; void *j; } xlnd_state; + + union + { + xlread_stateid c; + void *j; + } xlread_state; }; /* Get a new XLogReader */ -- 2.16.3 From 3c87303749fa4105d8dc40fe3dca2725aca34eaf Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 13:48:25 +0900 Subject: [PATCH 05/10] Make XLogPageRead not use callback but call the function directly This patch replaces the call to the callback in XLogReadRecrod with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/xlog.c | 44 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 59fd12153a..ba75ecb69c 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,13 +804,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */ static XLogSource currentSource = 0; /* XLOG_FROM_* code */ static bool lastSourceFailed = false; -typedef struct XLogPageReadPrivate -{ - int emode; - bool fetching_ckpt; /* are we fetching a checkpoint record? */ - bool randAccess; -} XLogPageReadPrivate; - /* * These variables track when we last obtained some WAL data to process, * and where we got it from. (XLogReceiptSource is initially the same as @@ -885,9 +878,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *readTLI); +static void XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, bool fetching_ckpt, XLogRecPtr tliRecPtr); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); @@ -4248,12 +4240,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, bool fetching_ckpt) { XLogRecord *record; - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - - /* Pass through parameters to XLogPageRead */ - private->fetching_ckpt = fetching_ckpt; - private->emode = emode; - private->randAccess = (RecPtr != InvalidXLogRecPtr); + bool randAccess = (RecPtr != InvalidXLogRecPtr); /* This is the first attempt to read this page. */ lastSourceFailed = false; @@ -4266,10 +4253,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, xlogreader->loadLen, - xlogreader->currRecPtr, xlogreader->readBuf, - &xlogreader->readPageTLI); + XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; @@ -6205,7 +6189,6 @@ StartupXLOG(void) bool backupFromStandby = false; DBState dbstate_at_startup; XLogReaderState *xlogreader; - XLogPageReadPrivate private; bool fast_promoted = false; struct stat st; @@ -6346,8 +6329,7 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - MemSet(&private, 0, sizeof(XLogPageReadPrivate)); - xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -11516,12 +11498,14 @@ CancelBackup(void) * sleep and retry. */ static void -XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI) +XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess) { - XLogPageReadPrivate *private = - (XLogPageReadPrivate *) xlogreader->private_data; - int emode = private->emode; + XLogRecPtr targetPagePtr = xlogreader->loadPagePtr; + int reqLen = xlogreader->loadLen; + XLogRecPtr targetRecPtr = xlogreader->currRecPtr; + char *readBuf = xlogreader->readBuf; + TimeLineID *readTLI = &xlogreader->readPageTLI; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11564,8 +11548,8 @@ retry: receivedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - private->randAccess, - private->fetching_ckpt, + randAccess, + fetching_ckpt, targetRecPtr)) { if (readFile >= 0) -- 2.16.3 From 02a8227dbbaf64f31667e3b5188c032be15923bb Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:02:19 +0900 Subject: [PATCH 06/10] Make XlogReadTwoPhaseData not use callback but call the function directly This patch replaces the call to the callback in XlogReadTwoPhaseData with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/twophase.c | 8 ++------ src/backend/access/transam/xlogutils.c | 8 +++++--- src/backend/replication/logical/logicalfuncs.c | 8 ++++++-- src/include/access/xlogutils.h | 5 +---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index a3573ad0af..caaa09d785 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1386,8 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, - NULL); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -1396,10 +1395,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, xlogreader->loadLen, - xlogreader->currRecPtr, xlogreader->readBuf, - &xlogreader->readPageTLI); + read_local_xlog_page(xlogreader); if (record == NULL) ereport(ERROR, diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index c853e1f0e3..fd461f16fc 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -908,10 +908,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * loop for now. */ void -read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page, - TimeLineID *pageTLI) +read_local_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *cur_page = state->readBuf; + TimeLineID *pageTLI = &state->readPageTLI; XLogRecPtr read_upto, loc; int count; diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 4d09255504..240a375d8f 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -118,8 +118,12 @@ void logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) { - read_local_xlog_page(state, targetPagePtr, reqLen, - targetRecPtr, cur_page, pageTLI); + Assert(targetPagePtr == state->loadPagePtr && + reqLen == state->loadLen && + targetRecPtr == state->currRecPtr && + cur_page == state->readBuf && + pageTLI == &state->readPageTLI); + read_local_xlog_page(state); } /* diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 5dba86b8b8..7f119837ce 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern void read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page, - TimeLineID *pageTLI); +extern void read_local_xlog_page(XLogReaderState *state); extern void XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength); -- 2.16.3 From 2b7e89d347c154a2bdb2e07b8c962df5e1a70e25 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:35:10 +0900 Subject: [PATCH 07/10] Make logical rep stuff not use callback but call the function directly This is a bit different from the two before. This patch moves the callback from XLogReaderState to LogicalDecodingContext. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/replication/logical/logical.c | 19 +++++++++---------- src/backend/replication/logical/logicalfuncs.c | 19 +++++-------------- src/backend/replication/slotfuncs.c | 9 +++------ src/backend/replication/walsender.c | 17 +++++++++-------- src/include/replication/logical.h | 13 +++++++++---- src/include/replication/logicalfuncs.h | 5 +---- 6 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 1740753b76..6dc87207b0 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -123,7 +123,7 @@ StartupDecodingContext(List *output_plugin_options, TransactionId xmin_horizon, bool need_full_snapshot, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -172,12 +172,14 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx); + ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->read_page = read_page; + ctx->reorder = ReorderBufferAllocate(); ctx->snapshot_builder = AllocateSnapshotBuilder(ctx->reorder, xmin_horizon, start_lsn, @@ -231,7 +233,7 @@ CreateInitDecodingContext(char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -373,7 +375,7 @@ LogicalDecodingContext * CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -474,6 +476,8 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) (uint32) (slot->data.restart_lsn >> 32), (uint32) slot->data.restart_lsn); + XLREAD_RESET(ctx->reader); + /* Wait for a consistent starting point */ for (;;) { @@ -483,12 +487,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) /* the read_page callback waits for new WAL */ while (XLogReadRecord(ctx->reader, startptr, &record, &err) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (err) elog(ERROR, "%s", err); diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 240a375d8f..0c4bc62cfa 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -115,15 +115,9 @@ check_permissions(void) } void -logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) +logical_read_local_xlog_page(LogicalDecodingContext *ctx) { - Assert(targetPagePtr == state->loadPagePtr && - reqLen == state->loadLen && - targetRecPtr == state->currRecPtr && - cur_page == state->readBuf && - pageTLI == &state->readPageTLI); - read_local_xlog_page(state); + read_local_xlog_page(ctx->reader); } /* @@ -286,6 +280,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin /* invalidate non-timetravel entries */ InvalidateSystemCaches(); + XLREAD_RESET(ctx->reader); + /* Decode until we run out of records */ while ((startptr != InvalidXLogRecPtr && startptr < end_of_wal) || (ctx->reader->EndRecPtr != InvalidXLogRecPtr && ctx->reader->EndRecPtr < end_of_wal)) @@ -295,12 +291,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin while (XLogReadRecord(ctx->reader, startptr, &record, &errm) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 4a8952931d..afdd3dea37 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -420,6 +420,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) /* invalidate non-timetravel entries */ InvalidateSystemCaches(); + XLREAD_RESET(ctx->reader); + /* Decode at least one record, until we run out of records */ while ((!XLogRecPtrIsInvalid(startlsn) && startlsn < moveto) || @@ -435,12 +437,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) */ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) == XLREAD_NEED_DATA) - ctx->reader->read_page(ctx->reader, - ctx->reader->loadPagePtr, - ctx->reader->loadLen, - ctx->reader->currRecPtr, - ctx->reader->readBuf, - &ctx->reader->readPageTLI); + ctx->read_page(ctx); if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index cb85ba3abf..5ac3ef4c66 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -762,9 +762,13 @@ StartReplication(StartReplicationCmd *cmd) * set every time WAL is flushed. */ static void -logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI) +logical_read_xlog_page(LogicalDecodingContext *ctx) { + XLogReaderState *state = ctx->reader; + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *cur_page = state->readBuf; + XLogRecPtr flushptr; int count; @@ -2821,14 +2825,11 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; + XLREAD_RESET(logical_decoding_ctx->reader); + while (XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm) == XLREAD_NEED_DATA) - logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader, - logical_decoding_ctx->reader->loadPagePtr, - logical_decoding_ctx->reader->loadLen, - logical_decoding_ctx->reader->currRecPtr, - logical_decoding_ctx->reader->readBuf, - &logical_decoding_ctx->reader->readPageTLI); + logical_decoding_ctx->read_page(logical_decoding_ctx); logical_startptr = InvalidXLogRecPtr; diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index 0a2a63a48c..70339eb8be 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -32,7 +32,11 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) ( TransactionId xid ); -typedef struct LogicalDecodingContext +typedef struct LogicalDecodingContext LogicalDecodingContext; + +typedef void (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx); + +struct LogicalDecodingContext { /* memory context this is all allocated in */ MemoryContext context; @@ -42,6 +46,7 @@ typedef struct LogicalDecodingContext /* infrastructure pieces for decoding */ XLogReaderState *reader; + LogicalDecodingXLogReadPageCB read_page; struct ReorderBuffer *reorder; struct SnapBuild *snapshot_builder; @@ -89,7 +94,7 @@ typedef struct LogicalDecodingContext bool prepared_write; XLogRecPtr write_location; TransactionId write_xid; -} LogicalDecodingContext; +}; extern void CheckLogicalDecodingRequirements(void); @@ -98,7 +103,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); @@ -106,7 +111,7 @@ extern LogicalDecodingContext *CreateDecodingContext( XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogPageReadCB read_page, + LogicalDecodingXLogReadPageCB read_page, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h index 3d00dee067..04a9fe10fa 100644 --- a/src/include/replication/logicalfuncs.h +++ b/src/include/replication/logicalfuncs.h @@ -11,9 +11,6 @@ #include "replication/logical.h" -extern void logical_read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, - char *cur_page, TimeLineID *pageTLI); +extern void logical_read_local_xlog_page(LogicalDecodingContext *ctx); #endif -- 2.16.3 From 3f8f71e23d6a453c969c0a0dbd764b9e19ef4f4b Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 15:50:51 +0900 Subject: [PATCH 08/10] Make pg_waldump not use callback but call the function directly This patch does the similar thing to the change in logical rep. Moves callback from XLogReaderState from the parameter of XLogFindNextRecord. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/backend/access/transam/xlogreader.c | 17 +++++++---------- src/bin/pg_waldump/pg_waldump.c | 21 +++++++++------------ src/include/access/xlogreader.h | 14 +++++--------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index f953924a72..9b38fc7829 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -996,7 +996,8 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * debugging purposes. */ XLogRecPtr -XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) +XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private) { XLogReaderState saved_state = *state; XLogRecPtr tmpRecPtr; @@ -1008,6 +1009,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) Assert(!XLogRecPtrIsInvalid(RecPtr)); + XLREAD_RESET(state); + /* * skip over potential continuation data, keeping in mind that it may span * multiple pages @@ -1035,9 +1038,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) /* Read the page containing the record */ while(XLogNeedData(state, targetPagePtr, targetRecOff)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); if (state->readLen < 0) goto err; @@ -1048,9 +1049,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) /* make sure we have enough data for the page header */ while (XLogNeedData(state, targetPagePtr, pageHeaderSize)) - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); if (state->readLen < 0) goto err; @@ -1097,9 +1096,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) { if (result == XLREAD_NEED_DATA) { - state->read_page(state, state->loadPagePtr, state->loadLen, - state->currRecPtr, state->readBuf, - &state->readPageTLI); + read_page(state, private); continue; } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index acee7ae199..1966da493f 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, * XLogReader read_page callback */ static void -XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI) +XLogDumpReadPage(XLogReaderState *state, void *priv) { - XLogDumpPrivate *private = state->private_data; + XLogRecPtr targetPagePtr = state->loadPagePtr; + int reqLen = state->loadLen; + char *readBuff = state->readBuf; + XLogDumpPrivate *private = (XLogDumpPrivate *) priv; int count = XLOG_BLCKSZ; if (private->endptr != InvalidXLogRecPtr) @@ -1095,13 +1097,13 @@ main(int argc, char **argv) /* done with argument parsing, do the actual work */ /* we have everything we need, start reading */ - xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage, - &private); + xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL); if (!xlogreader_state) fatal_error("out of memory"); /* first find a valid recptr to start from */ - first_record = XLogFindNextRecord(xlogreader_state, private.startptr); + first_record = XLogFindNextRecord(xlogreader_state, private.startptr, + &XLogDumpReadPage, (void*) &private); if (first_record == InvalidXLogRecPtr) fatal_error("could not find a valid record after %X/%X", @@ -1128,12 +1130,7 @@ main(int argc, char **argv) while (XLogReadRecord(xlogreader_state, first_record, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader_state->read_page(xlogreader_state, - xlogreader_state->loadPagePtr, - xlogreader_state->loadLen, - xlogreader_state->currRecPtr, - xlogreader_state->readBuf, - &xlogreader_state->readPageTLI); + XLogDumpReadPage(xlogreader_state, (void *) &private); if (!record) { diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 9bfa9e8d54..b231cb330c 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -29,14 +29,6 @@ typedef struct XLogReaderState XLogReaderState; -/* Function type definition for the read_page callback */ -typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, - XLogRecPtr targetRecPtr, - char *readBuf, - TimeLineID *pageTLI); - typedef struct { /* Is this block ref in use? */ @@ -263,7 +255,11 @@ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, extern void XLogReaderInvalReadState(XLogReaderState *state); #ifdef FRONTEND -extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); +/* Function type definition for the read_page callback */ +typedef void (*XLogFindNextRecordCB) (XLogReaderState *xlogreader, + void *private); +extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private); #endif /* FRONTEND */ /* Functions for decoding an XLogRecord */ -- 2.16.3 From e37e37273ccaeb4d135d1859fb9a068bd0b64340 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 16:00:57 +0900 Subject: [PATCH 09/10] Make pg_rewind not use callback but call the function directly This patch replaces the call to the callback in pg_rewind with direct call to the original function. Then invalidate the parameters callback and private for XLogReaderAllocate. --- src/bin/pg_rewind/parsexlog.c | 76 ++++++++++++------------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 21b638ed34..f28f5d71f5 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -42,16 +42,8 @@ static int xlogreadfd = -1; static XLogSegNo xlogreadsegno = -1; static char xlogfpath[MAXPGPATH]; -typedef struct XLogPageReadPrivate -{ - const char *datadir; - int tliIndex; -} XLogPageReadPrivate; - static void SimpleXLogPageRead(XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI); + const char *datadir, int *tliIndex); /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline @@ -65,12 +57,8 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; - private.datadir = datadir; - private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -78,12 +66,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, { while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, datadir, &tliIndex); if (record == NULL) { @@ -125,24 +108,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; XLogRecPtr endptr; - private.datadir = datadir; - private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, datadir, &tliIndex); if (record == NULL) { @@ -178,7 +152,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr searchptr; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; /* * The given fork pointer points to the end of the last common record, @@ -194,10 +167,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - private.datadir = datadir; - private.tliIndex = tliIndex; - xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, - &private); + xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -208,12 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) == XLREAD_NEED_DATA) - xlogreader->read_page(xlogreader, - xlogreader->loadPagePtr, - xlogreader->loadLen, - xlogreader->currRecPtr, - xlogreader->readBuf, - &xlogreader->readPageTLI); + SimpleXLogPageRead(xlogreader, datadir, &tliIndex); if (record == NULL) { @@ -260,11 +225,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, /* XLogreader callback function, to read a WAL page */ static void -SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI) +SimpleXLogPageRead(XLogReaderState *xlogreader, + const char*datadir, int *tliIndex) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogRecPtr targetPagePtr = xlogreader->loadPagePtr; + char *readBuf = xlogreader->readBuf; + TimeLineID *pageTLI = &xlogreader->readPageTLI; uint32 targetPageOff; XLogRecPtr targetSegEnd; XLogSegNo targetSegNo; @@ -297,17 +263,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, * be done both forward and backward, consider also switching timeline * accordingly. */ - while (private->tliIndex < targetNentries - 1 && - targetHistory[private->tliIndex].end < targetSegEnd) - private->tliIndex++; - while (private->tliIndex > 0 && - targetHistory[private->tliIndex].begin >= targetSegEnd) - private->tliIndex--; + while (*tliIndex < targetNentries - 1 && + targetHistory[*tliIndex].end < targetSegEnd) + (*tliIndex)++; + while (*tliIndex > 0 && + targetHistory[*tliIndex].begin >= targetSegEnd) + (*tliIndex)--; - XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, + XLogFileName(xlogfname, targetHistory[*tliIndex].tli, xlogreadsegno, WalSegSz); - snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname); + snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", datadir, xlogfname); xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); @@ -348,7 +314,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, Assert(targetSegNo == xlogreadsegno); - *pageTLI = targetHistory[private->tliIndex].tli; + *pageTLI = targetHistory[*tliIndex].tli; xlogreader->readLen = XLOG_BLCKSZ; return; -- 2.16.3 From 6bbdb4ac24f1a9e253d05c5ed0c0e98477834026 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp> Date: Thu, 18 Apr 2019 16:15:18 +0900 Subject: [PATCH 10/10] Remove callback entry from XLogReaderState This is the third (3/2?) step. Remove no-longer-useful members read_page and private_data from XLogReaderState. Then change XLogReaderAllocate not to take the parameters. --- src/backend/access/transam/twophase.c | 2 +- src/backend/access/transam/xlog.c | 4 ++-- src/backend/access/transam/xlogreader.c | 9 ++------- src/backend/replication/logical/logical.c | 2 +- src/bin/pg_rewind/parsexlog.c | 6 +++--- src/bin/pg_waldump/pg_waldump.c | 2 +- src/include/access/xlogreader.h | 32 +------------------------------ 7 files changed, 11 insertions(+), 46 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index caaa09d785..e70245e7cb 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1386,7 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + xlogreader = XLogReaderAllocate(wal_segment_size); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index ba75ecb69c..f34a9ba806 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1188,7 +1188,7 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + debug_reader = XLogReaderAllocate(wal_segment_size); if (!debug_reader) { @@ -6329,7 +6329,7 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + xlogreader = XLogReaderAllocate(wal_segment_size); if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 9b38fc7829..9219314d93 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -118,8 +118,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) * Returns NULL if the xlogreader couldn't be allocated. */ XLogReaderState * -XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc, - void *private_data) +XLogReaderAllocate(int wal_segment_size) { XLogReaderState *state; @@ -147,11 +146,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc, } state->wal_segment_size = wal_segment_size; - state->read_page = pagereadfunc; - /* system_identifier initialized to zeroes above */ - state->private_data = private_data; - /* ReadRecPtr and EndRecPtr initialized to zeroes above */ - /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */ + /* All members are initialized to zeroes above */ state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1, MCXT_ALLOC_NO_OOM); if (!state->errormsg_buf) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 6dc87207b0..e257dd8afc 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -172,7 +172,7 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); + ctx->reader = XLogReaderAllocate(wal_segment_size); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index f28f5d71f5..b8b4bc2acd 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogReaderState *xlogreader; char *errormsg; - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -110,7 +110,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) char *errormsg; XLogRecPtr endptr; - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -167,7 +167,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader = XLogReaderAllocate(WalSegSz); if (xlogreader == NULL) pg_fatal("out of memory"); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 1966da493f..f70dc7bbc0 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -1097,7 +1097,7 @@ main(int argc, char **argv) /* done with argument parsing, do the actual work */ /* we have everything we need, start reading */ - xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL); + xlogreader_state = XLogReaderAllocate(WalSegSz); if (!xlogreader_state) fatal_error("out of memory"); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b231cb330c..262bf0e77f 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -98,40 +98,12 @@ struct XLogReaderState */ int wal_segment_size; - /* - * Data input callback (mandatory). - * - * This callback shall read at least reqLen valid bytes of the xlog page - * starting at targetPagePtr, and store them in readBuf. The callback - * shall return the number of bytes read (never more than XLOG_BLCKSZ), or - * -1 on failure. The callback shall sleep, if necessary, to wait for the - * requested bytes to become available. The callback will not be invoked - * again for the same page unless more than the returned number of bytes - * are needed. - * - * targetRecPtr is the position of the WAL record we're reading. Usually - * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs - * to read and verify the page or segment header, before it reads the - * actual WAL record it's interested in. In that case, targetRecPtr can - * be used to determine which timeline to read the page from. - * - * The callback shall set *pageTLI to the TLI of the file the page was - * read from. It is currently used only for error reporting purposes, to - * reconstruct the name of the WAL file where an error occurred. - */ - XLogPageReadCB read_page; - /* * System identifier of the xlog files we're about to read. Set to zero * (the default value) if unknown or unimportant. */ uint64 system_identifier; - /* - * Opaque data for callbacks to use. Not used by XLogReader. - */ - void *private_data; - /* * Start and end point of last record read. EndRecPtr is also used as the * position to read next, if XLogReadRecord receives an invalid recptr. @@ -236,9 +208,7 @@ struct XLogReaderState }; /* Get a new XLogReader */ -extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, - XLogPageReadCB pagereadfunc, - void *private_data); +extern XLogReaderState *XLogReaderAllocate(int wal_segment_size); /* Free an XLogReader */ extern void XLogReaderFree(XLogReaderState *state); -- 2.16.3
Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> wrote:
> Hello. Thank you for looking this.
> ...
> Yeah, I'll register this, maybe the week after next week.
I've checked the new version. One more thing I noticed now is that XLR_STATE.j
is initialized to zero, either by XLogReaderAllocate() which zeroes the whole
reader state, or later by XLREAD_RESET. This special value then needs to be
handled here:
#define XLR_SWITCH() \
do { \
if ((XLR_STATE).j) \
goto *((void *) (XLR_STATE).j); \
XLR_CASE(XLR_INIT_STATE); \
} while (0)
I think it's better to set the label always to (&&XLR_INIT_STATE) so that
XLR_SWITCH can perform the jump unconditionally.
Attached is also an (unrelated) comment fix proposal.
--
Antonin Houska
Web: https://www.cybertec-postgresql.com
Вложения
Thank you for looking this, Antonin.
At Wed, 22 May 2019 13:53:23 +0200, Antonin Houska <ah@cybertec.at> wrote in <25494.1558526003@spoje.net>
> Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> wrote:
>
> > Hello. Thank you for looking this.
> > ...
> > Yeah, I'll register this, maybe the week after next week.
>
> I've checked the new version. One more thing I noticed now is that XLR_STATE.j
> is initialized to zero, either by XLogReaderAllocate() which zeroes the whole
> reader state, or later by XLREAD_RESET. This special value then needs to be
> handled here:
>
> #define XLR_SWITCH() \
> do { \
> if ((XLR_STATE).j) \
> goto *((void *) (XLR_STATE).j); \
> XLR_CASE(XLR_INIT_STATE); \
> } while (0)
>
> I think it's better to set the label always to (&&XLR_INIT_STATE) so that
> XLR_SWITCH can perform the jump unconditionally.
I thought the same but did not do that since label is
function-scoped so it cannot be referred outside the defined
function.
I moved the state variable from XLogReaderState into functions
static variable. It's not problem since the functions are
non-reentrant in the first place.
> Attached is also an (unrelated) comment fix proposal.
Sounds reasoable. I found another typo "acutually" there.
- int32 readLen; /* bytes acutually read, must be larger than
+ int32 readLen; /* bytes acutually read, must be at least
I fixed it with other typos found.
v3-0001 : Changed macrosas suggested.
v3-0002, 0004: Fixed comments. Fixes following changes of
macros. Renamed state symbols.
v3-0003, 0005-0010: No substantial change from v2.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 036b1a50ad79ebfde990e178bead9ba03c753d02 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Wed, 17 Apr 2019 14:15:16 +0900
Subject: [PATCH 02/10] Make ReadPageInternal a state machine
This patch set aims to remove read_page call back from
XLogReaderState. This is done in two steps. This patch does the first
step. Makes ReadPageInternal, which currently calling read_page
callback, a state machine and move the caller sites to XLogReadRecord
and other direct callers of the callback.
---
src/backend/access/transam/xlog.c | 15 ++-
src/backend/access/transam/xlogreader.c | 180 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 8 +-
src/backend/replication/logical/logicalfuncs.c | 6 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 ++-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 39 ++++--
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 177 insertions(+), 110 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1c7dd51b9f..d1a29fe5cd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -885,7 +885,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -11507,7 +11507,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static void
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11566,7 +11566,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
}
@@ -11661,7 +11662,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11675,8 +11677,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index baf04b9f1f..f7499aff5e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -118,8 +118,8 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen);
static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3);
static void ResetDecoder(XLogReaderState *state);
@@ -306,7 +306,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -358,15 +357,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -392,8 +393,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -470,14 +471,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(total_len - gotlen + SizeOfXLogShortPHD,
+ XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
- if (readOff < 0)
+ if (state->readLen < 0)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -506,20 +510,28 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
+ if (state->readLen < pageHeaderSize)
+ {
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ }
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
+ if (state->readLen < pageHeaderSize + len)
+ {
+ if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ }
memcpy(buffer, (char *) contdata, len);
buffer += len;
@@ -550,9 +562,13 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/* Record does not cross a page boundary */
@@ -595,39 +611,48 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the required data is fully loaded. state->readLen is set to
+ * -1 when the loaded data is found to be invalid.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise, returns true and requests data using state->loadPagePtr and
+ * state->loadLen. The caller should load the region to state->readBuf and
+ * call this function again.
+ *
+ * Note: This function is not reentrant. The state is maintained internally in
+ * the function. DO NOT ereport(ERROR)-out from inside of this function.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
{
- int readLen;
- uint32 targetPageOff;
- XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ /*
+ * This function is a state machine that can exit and reenter at any place
+ * marked as XLR_LEAVE. All internal state is preserved through multiple
+ * calls.
+ */
+ static uint32 targetPageOff;
+ static XLogSegNo targetSegNo;
+ static XLogPageHeader hdr;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ XLR_SWITCH (XLND_STATE_INIT);
XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
/* check whether we have all the requested data already */
if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
reqLen <= state->readLen)
- return state->readLen;
+ XLR_RETURN(false);
/*
* Data is not in our buffer.
*
* Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
+ * before, we need to do verification as the caller might now be rereading
+ * data from a different source.
*
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
@@ -636,18 +661,17 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ state->loadPagePtr = pageptr - targetPageOff;
+ state->loadLen = XLOG_BLCKSZ;
+ XLR_LEAVE(XLND_STATE_SEGHEADER, true);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ if (state->readLen < 0)
goto err;
/* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ Assert(state->readLen == XLOG_BLCKSZ);
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
+ if (!XLogReaderValidatePageHeader(state, state->loadPagePtr,
state->readBuf))
goto err;
}
@@ -656,48 +680,53 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
* First, read the requested data length, but at least a short page header
* so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ state->loadPagePtr = pageptr;
+ state->loadLen = Max(reqLen, SizeOfXLogShortPHD);
+ XLR_LEAVE(XLND_STATE_PAGEHEADER, true);
+
+ if (state->readLen < 0)
goto err;
- Assert(readLen <= XLOG_BLCKSZ);
+ Assert(state->readLen <= XLOG_BLCKSZ);
/* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
+ if (state->readLen <= SizeOfXLogShortPHD)
goto err;
- Assert(readLen >= reqLen);
+ Assert(state->readLen >= state->loadLen);
hdr = (XLogPageHeader) state->readBuf;
/* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
+ if (state->readLen < XLogPageHeaderSize(hdr))
{
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ state->loadPagePtr = pageptr;
+ state->loadLen = XLogPageHeaderSize(hdr);
+ XLR_LEAVE(XLND_STATE_PAGELONGHEADER, true);
+
+ if (state->readLen < 0)
goto err;
}
+ XLR_SWITCH_END();
+
/*
* Now that we know we have the full header, validate it.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
+ if (!XLogReaderValidatePageHeader(state, pageptr, (char *) state->readBuf))
+ goto err;
/* update read state information */
state->readSegNo = targetSegNo;
state->readOff = targetPageOff;
- state->readLen = readLen;
- return readLen;
+ XLR_RETURN(false);
err:
XLogReaderInvalReadState(state);
- return -1;
+ state->readLen = -1;
+
+ XLR_RETURN(false);
}
/*
@@ -996,7 +1025,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -1004,7 +1032,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -1013,8 +1041,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
targetPagePtr = tmpRecPtr - targetRecOff;
/* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
header = (XLogPageHeader) state->readBuf;
@@ -1022,8 +1054,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
pageHeaderSize = XLogPageHeaderSize(header);
/* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/* skip over potential continuation data */
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..c853e1f0e3 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+void
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..16c6095179 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+void
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ read_local_xlog_page(state, targetPagePtr, reqLen,
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 3f31368022..082c98b2ec 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static void
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 287af60c4e..0bfbea8f09 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static void SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogreader callback function, to read a WAL page */
-static int
+static void
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -291,7 +291,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
}
@@ -304,7 +305,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
@@ -317,13 +319,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c40014d483..c748820a5f 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static void
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 04228e2a87..d3b3e4b7ba 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -30,7 +30,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -66,6 +66,15 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* internal state of XLogNeedData() */
+typedef enum xlnd_stateid
+{
+ XLND_STATE_INIT,
+ XLND_STATE_SEGHEADER,
+ XLND_STATE_PAGEHEADER,
+ XLND_STATE_PAGELONGHEADER
+} xlnd_stateid;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -120,6 +129,22 @@ struct XLogReaderState
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* parameters to page reader */
+ XLogRecPtr loadPagePtr; /* Pointer to the page */
+ int loadLen; /* wanted length in bytes */
+ char *readBuf; /* buffer to store data */
+ XLogRecPtr currRecPtr; /* beginning of the WAL record being read */
+
+ /* return from page reader */
+ int32 readLen; /* bytes actually read, must be at least
+ * loadLen. -1 on error. */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+
/* ----------------------------------------
* Decoded representation of current record
*
@@ -145,17 +170,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
XLogSegNo readSegNo;
uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
@@ -164,8 +181,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..be0c79d18c 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern void read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..b0306c54ab 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern void logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
From 94bf31fb10ce24284c27b49b6dcc206ebf9344cc Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Fri, 24 May 2019 09:22:44 +0900
Subject: [PATCH 03/10] Change interface of XLogReadRecord
As a preparation to the second step, this patch changes the interface
of XLogReadRecord so that the function can request the callers for
reading more data.
---
src/backend/access/transam/twophase.c | 2 +-
src/backend/access/transam/xlog.c | 2 +-
src/backend/access/transam/xlogreader.c | 52 ++++++++++++++------------
src/backend/replication/logical/logical.c | 2 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/slotfuncs.c | 2 +-
src/backend/replication/walsender.c | 2 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 13 ++++++-
10 files changed, 49 insertions(+), 36 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5196d6181d..334abe7ba8 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1394,7 +1394,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ XLogReadRecord(xlogreader, lsn, &record, &errormsg);
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index d1a29fe5cd..ee73576d5b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4262,7 +4262,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
{
char *errormsg;
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
+ XLogReadRecord(xlogreader, RecPtr, &record, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f7499aff5e..ee11642759 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -285,20 +285,21 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
* If RecPtr is valid, try to read a record at that position. Otherwise
* try to read a record just after the last one previously read.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * If the read_page callback fails to read the requested data, *record is set
+ * to NULL and XLREAD_FAIL is returned. The callback is expected to have
+ * reported the error; errormsg is set to NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * If the reading fails for some other reason, *record is also set to NULL and
+ * XLREAD_FAIL is returned. *errormsg is set to a string with details of the
+ * failure.
*
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogRecord **record, char **errormsg)
{
- XLogRecord *record;
XLogRecPtr targetPagePtr;
bool randAccess;
uint32 len,
@@ -405,8 +406,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* cannot access any other fields until we've verified that we got the
* whole header.
*/
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ *record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
+ total_len = (*record)->xl_tot_len;
/*
* If the whole record header is on this page, validate it immediately.
@@ -418,7 +419,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
*/
if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
{
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
+ if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, *record,
randAccess))
goto err;
gotheader = true;
@@ -540,9 +541,9 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* If we just reassembled the record header, validate it. */
if (!gotheader)
{
- record = (XLogRecord *) state->readRecordBuf;
+ *record = (XLogRecord *) state->readRecordBuf;
if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ *record, randAccess))
goto err;
gotheader = true;
}
@@ -550,8 +551,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
+ *record = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, *record, RecPtr))
goto err;
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
@@ -572,7 +573,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
/* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
+ if (!ValidXLogRecord(state, *record, RecPtr))
goto err;
state->EndRecPtr = RecPtr + MAXALIGN(total_len);
@@ -583,18 +584,19 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->wal_segment_size - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
@@ -607,7 +609,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -1011,6 +1014,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1099,7 +1103,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index bbd38c06d1..79a3ec09ff 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -481,7 +481,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ XLogReadRecord(ctx->reader, startptr, &record, &err);
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 16c6095179..6fc78d7445 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -289,7 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ XLogReadRecord(ctx->reader, startptr, &record, &errm);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 808a6f5b83..7db8e0d044 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ XLogReadRecord(ctx->reader, startlsn, &record, &errm);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 082c98b2ec..d136a48805 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2826,7 +2826,7 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm);
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0bfbea8f09..512005de1c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -75,7 +75,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ XLogReadRecord(xlogreader, startpoint, &record, &errormsg);
if (record == NULL)
{
@@ -127,7 +127,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ XLogReadRecord(xlogreader, ptr, &record, &errormsg);
if (record == NULL)
{
if (errormsg)
@@ -190,7 +190,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ XLogReadRecord(xlogreader, searchptr, &record, &errormsg);
if (record == NULL)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c748820a5f..3198627242 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1125,7 +1125,7 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ XLogReadRecord(xlogreader_state, first_record, &record, &errormsg);
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index d3b3e4b7ba..0e734d27f1 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -66,6 +66,13 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
/* internal state of XLogNeedData() */
typedef enum xlnd_stateid
{
@@ -220,8 +227,10 @@ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
extern void XLogReaderFree(XLogReaderState *state);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
--
2.16.3
From 6d3694958c65e1fb930bc858d75f0499af788770 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Fri, 24 May 2019 09:33:29 +0900
Subject: [PATCH 04/10] Make XLogReadRecord a state machine
This patch moves the caller sites of the callback above XLogReadRecord
by making XLogReadRecord a state machine.
---
src/backend/access/transam/twophase.c | 8 +-
src/backend/access/transam/xlog.c | 8 +-
src/backend/access/transam/xlogreader.c | 108 ++++++++++++++++---------
src/backend/replication/logical/logical.c | 10 ++-
src/backend/replication/logical/logicalfuncs.c | 10 ++-
src/backend/replication/slotfuncs.c | 10 ++-
src/backend/replication/walsender.c | 10 ++-
src/bin/pg_rewind/parsexlog.c | 28 ++++++-
src/bin/pg_waldump/pg_waldump.c | 11 ++-
src/include/access/xlogreader.h | 12 +++
10 files changed, 165 insertions(+), 50 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 334abe7ba8..ad0d505e54 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1394,7 +1394,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- XLogReadRecord(xlogreader, lsn, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr, xlogreader->loadLen,
+ xlogreader->currRecPtr, xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ee73576d5b..5b9b1893a2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4262,7 +4262,13 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
{
char *errormsg;
- XLogReadRecord(xlogreader, RecPtr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg)
+ == XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr, xlogreader->loadLen,
+ xlogreader->currRecPtr, xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ee11642759..ce901c358a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -282,31 +282,57 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * This function runs a state machine and may need to call several times until
+ * a record is read.
*
- * If the read_page callback fails to read the requested data, *record is set
- * to NULL and XLREAD_FAIL is returned. The callback is expected to have
- * reported the error; errormsg is set to NULL.
+ * At the initial state, if called with valid pRecPtr, try to read a record at
+ * that position. If invalid pRecPtr is given try to read a record just after
+ * the last one previously read.
*
- * If the reading fails for some other reason, *record is also set to NULL and
- * XLREAD_FAIL is returned. *errormsg is set to a string with details of the
- * failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record then the state machine is reset to initial
+ * state.
+ *
+ * Returns XLREAD_NEED_DATA if needs more data fed. In that case loadPagePtr
+ * and loadLen in state is set to inform the required WAL data. The caller
+ * shall read in the requested data into readBuf and set readLen and
+ * readPageTLI to the length of the data actually read and the TLI for the
+ * data read in respectively. In case of failure the caller shall call the
+ * function setting readLen to -1 and storing error message in errormsg_buf to
+ * inform error.
+ *
+ * If the reading fails for some reasons including caller-side error mentioned
+ * above, returns XLREAD_FAIL with *record being set to NULL. *errormsg is set
+ * to a string with details of the failure. The state machine is reset to
+ * initial state.
*
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
+ *
+ * Note: This function is not reentrant. The state is maintained internally in
+ * the function. DO NOT non-local exit (ereport) from inside of this function.
*/
XLogReadRecordResult
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+XLogReadRecord(XLogReaderState *state, XLogRecPtr pRecPtr,
XLogRecord **record, char **errormsg)
{
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ /*
+ * This function is a state machine that can exit and reenter at any place
+ * marked as XLR_LEAVE. All internal state needs to be preserved through
+ * multiple calls.
+ */
+ static XLogRecPtr targetPagePtr;
+ static bool randAccess;
+ static uint32 len,
+ total_len;
+ static uint32 targetRecOff;
+ static uint32 pageHeaderSize;
+ static bool gotheader;
+ static XLogRecPtr RecPtr;
+
+ XLR_SWITCH(XLREAD_STATE_INIT);
+
+ RecPtr = pRecPtr;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -360,10 +386,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*/
while (XLogNeedData(state, targetPagePtr,
Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
-
+ XLR_LEAVE(XLREAD_STATE_PAGE, XLREAD_NEED_DATA);
+
if (state->readLen < 0)
goto err;
@@ -442,10 +466,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (total_len > len)
{
/* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ static char *contdata;
+ static XLogPageHeader pageHeader;
+ static char *buffer;
+ static uint32 gotlen;
/*
* Enlarge readRecordBuf as needed.
@@ -475,9 +499,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
while (XLogNeedData(state, targetPagePtr,
Min(total_len - gotlen + SizeOfXLogShortPHD,
XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTPAGE, XLREAD_NEED_DATA);
if (state->readLen < 0)
goto err;
@@ -514,9 +536,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (state->readLen < pageHeaderSize)
{
while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTPAGE_HEADER, XLREAD_NEED_DATA);
}
Assert(pageHeaderSize <= state->readLen);
@@ -529,9 +549,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (state->readLen < pageHeaderSize + len)
{
if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTRECORD, XLREAD_NEED_DATA);
}
memcpy(buffer, (char *) contdata, len);
@@ -565,9 +583,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
/* Wait for the record data to become available */
while (XLogNeedData(state, targetPagePtr,
Min(targetRecOff + total_len, XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_RECORD, XLREAD_NEED_DATA);
if (state->readLen < 0)
goto err;
@@ -581,6 +597,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
state->ReadRecPtr = RecPtr;
}
+ XLR_SWITCH_END();
+
/*
* Special processing if it's an XLOG SWITCH record
*/
@@ -593,10 +611,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
}
if (DecodeXLogRecord(state, *record, errormsg))
- return XLREAD_SUCCESS;
+ XLR_RETURN(XLREAD_SUCCESS);
*record = NULL;
- return XLREAD_FAIL;
+ XLR_RETURN(XLREAD_FAIL);
err:
@@ -610,7 +628,7 @@ err:
*errormsg = state->errormsg_buf;
*record = NULL;
- return XLREAD_FAIL;
+ XLR_RETURN(XLREAD_FAIL);
}
/*
@@ -1015,6 +1033,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1103,8 +1122,17 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ continue;
+ }
+
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 79a3ec09ff..108dd6f9b2 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -481,7 +481,15 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- XLogReadRecord(ctx->reader, startptr, &record, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 6fc78d7445..4d09255504 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -289,7 +289,15 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- XLogReadRecord(ctx->reader, startptr, &record, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 7db8e0d044..f4f4a907ad 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,15 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- XLogReadRecord(ctx->reader, startlsn, &record, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d136a48805..e1c721bc9e 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2826,7 +2826,15 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader,
+ logical_decoding_ctx->reader->loadPagePtr,
+ logical_decoding_ctx->reader->loadLen,
+ logical_decoding_ctx->reader->currRecPtr,
+ logical_decoding_ctx->reader->readBuf,
+ &logical_decoding_ctx->reader->readPageTLI);
+
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 512005de1c..e26127206c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -75,7 +75,14 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
do
{
- XLogReadRecord(xlogreader, startpoint, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
if (record == NULL)
{
@@ -127,7 +134,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
if (xlogreader == NULL)
pg_fatal("out of memory");
- XLogReadRecord(xlogreader, ptr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
if (record == NULL)
{
if (errormsg)
@@ -190,7 +205,14 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- XLogReadRecord(xlogreader, searchptr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
if (record == NULL)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 3198627242..54717c9320 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1125,7 +1125,16 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- XLogReadRecord(xlogreader_state, first_record, &record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader_state->read_page(xlogreader_state,
+ xlogreader_state->loadPagePtr,
+ xlogreader_state->loadLen,
+ xlogreader_state->currRecPtr,
+ xlogreader_state->readBuf,
+ &xlogreader_state->readPageTLI);
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0e734d27f1..bc0c642906 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -70,6 +70,7 @@ typedef struct
typedef enum XLogReadRecordResult
{
XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
XLREAD_FAIL /* failed during reading a record */
} XLogReadRecordResult;
@@ -82,6 +83,17 @@ typedef enum xlnd_stateid
XLND_STATE_PAGELONGHEADER
} xlnd_stateid;
+/* internal state of XLogReadRecord() */
+typedef enum xlread_stateid
+{
+ XLREAD_STATE_INIT,
+ XLREAD_STATE_PAGE,
+ XLREAD_STATE_CONTPAGE,
+ XLREAD_STATE_CONTPAGE_HEADER,
+ XLREAD_STATE_CONTRECORD,
+ XLREAD_STATE_RECORD
+} xlread_stateid;
+
struct XLogReaderState
{
/* ----------------------------------------
--
2.16.3
From fce5b4fa23442e10d96dce648ce7c477476cbbe8 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 13:48:25 +0900
Subject: [PATCH 05/10] Make XLogPageRead not use callback but call the
function directly
This patch replaces the call to the callback in XLogReadRecord with
direct call to the original function. Then invalidate the parameters
callback and private for XLogReaderAllocate.
---
src/backend/access/transam/xlog.c | 44 +++++++++++++--------------------------
1 file changed, 14 insertions(+), 30 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5b9b1893a2..9f184eefbe 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -804,13 +804,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -885,9 +878,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *readTLI);
+static void XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -4248,12 +4240,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- /* Pass through parameters to XLogPageRead */
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4264,10 +4251,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg)
== XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr, xlogreader->loadLen,
- xlogreader->currRecPtr, xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
@@ -6203,7 +6187,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6344,8 +6327,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11514,12 +11496,14 @@ CancelBackup(void)
* sleep and retry.
*/
static void
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ XLogRecPtr targetPagePtr = xlogreader->loadPagePtr;
+ int reqLen = xlogreader->loadLen;
+ XLogRecPtr targetRecPtr = xlogreader->currRecPtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *readTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11562,8 +11546,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
--
2.16.3
From b9e2755f2e52f981255f1123f24922cc29e308e8 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:02:19 +0900
Subject: [PATCH 06/10] Make XlogReadTwoPhaseData not use callback but call the
function directly
This patch replaces the call to the callback in XlogReadTwoPhaseData
with direct call to the original function. Then invalidate the
parameters callback and private for XLogReaderAllocate.
---
src/backend/access/transam/twophase.c | 8 ++------
src/backend/access/transam/xlogutils.c | 8 +++++---
src/backend/replication/logical/logicalfuncs.c | 8 ++++++--
src/include/access/xlogutils.h | 5 +----
4 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ad0d505e54..23424886dc 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1386,8 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page,
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1396,10 +1395,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr, xlogreader->loadLen,
- xlogreader->currRecPtr, xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ read_local_xlog_page(xlogreader);
if (record == NULL)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index c853e1f0e3..fd461f16fc 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -908,10 +908,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
void
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *cur_page = state->readBuf;
+ TimeLineID *pageTLI = &state->readPageTLI;
XLogRecPtr read_upto,
loc;
int count;
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 4d09255504..240a375d8f 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -118,8 +118,12 @@ void
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
- read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ Assert(targetPagePtr == state->loadPagePtr &&
+ reqLen == state->loadLen &&
+ targetRecPtr == state->currRecPtr &&
+ cur_page == state->readBuf &&
+ pageTLI == &state->readPageTLI);
+ read_local_xlog_page(state);
}
/*
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index be0c79d18c..9724ce20b8 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern void read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI);
+extern void read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
--
2.16.3
From 39151062558d909df02dfedd4f5183f4f62c02ed Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:35:10 +0900
Subject: [PATCH 07/10] Make logical rep stuff not use callback but call the
function directly
This is a bit different from the two before. This patch moves the
callback from XLogReaderState to LogicalDecodingContext. Then
invalidate the parameters callback and private for XLogReaderAllocate.
---
src/backend/replication/logical/logical.c | 17 +++++++----------
src/backend/replication/logical/logicalfuncs.c | 17 +++--------------
src/backend/replication/slotfuncs.c | 7 +------
src/backend/replication/walsender.c | 15 +++++++--------
src/include/replication/logical.h | 13 +++++++++----
src/include/replication/logicalfuncs.h | 5 +----
6 files changed, 28 insertions(+), 46 deletions(-)
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 108dd6f9b2..767d73f476 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -123,7 +123,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -172,12 +172,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
+
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
AllocateSnapshotBuilder(ctx->reorder, xmin_horizon, start_lsn,
@@ -231,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -373,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -483,12 +485,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
/* the read_page callback waits for new WAL */
while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (err)
elog(ERROR, "%s", err);
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 240a375d8f..2a3f6d3cde 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,15 +115,9 @@ check_permissions(void)
}
void
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- Assert(targetPagePtr == state->loadPagePtr &&
- reqLen == state->loadLen &&
- targetRecPtr == state->currRecPtr &&
- cur_page == state->readBuf &&
- pageTLI == &state->readPageTLI);
- read_local_xlog_page(state);
+ read_local_xlog_page(ctx->reader);
}
/*
@@ -295,12 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f4f4a907ad..8675c9203e 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -435,12 +435,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
*/
while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e1c721bc9e..fde94a561f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,9 +762,13 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static void
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *cur_page = state->readBuf;
+
XLogRecPtr flushptr;
int count;
@@ -2828,12 +2832,7 @@ XLogSendLogical(void)
while (XLogReadRecord(logical_decoding_ctx->reader,
logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
- logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader,
- logical_decoding_ctx->reader->loadPagePtr,
- logical_decoding_ctx->reader->loadLen,
- logical_decoding_ctx->reader->currRecPtr,
- logical_decoding_ctx->reader->readBuf,
- &logical_decoding_ctx->reader->readPageTLI);
+ logical_decoding_ctx->read_page(logical_decoding_ctx);
logical_startptr = InvalidXLogRecPtr;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index f76b9eb4df..f0bec63241 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -32,7 +32,11 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (
TransactionId xid
);
-typedef struct LogicalDecodingContext
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef void (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
+struct LogicalDecodingContext
{
/* memory context this is all allocated in */
MemoryContext context;
@@ -42,6 +46,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -89,7 +94,7 @@ typedef struct LogicalDecodingContext
bool prepared_write;
XLogRecPtr write_location;
TransactionId write_xid;
-} LogicalDecodingContext;
+};
extern void CheckLogicalDecodingRequirements(void);
@@ -98,7 +103,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
@@ -106,7 +111,7 @@ extern LogicalDecodingContext *CreateDecodingContext(
XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index b0306c54ab..04a9fe10fa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern void logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page, TimeLineID *pageTLI);
+extern void logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From 19765979c323afc0f45b507fea6b3187274c6dc8 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:50:51 +0900
Subject: [PATCH 08/10] Make pg_waldump not use callback but call the function
directly
This patch does the similar thing to the change in logical rep. Moves
callback from XLogReaderState from the parameter of
XLogFindNextRecord. Then invalidate the parameters callback and
private for XLogReaderAllocate.
---
src/backend/access/transam/xlogreader.c | 15 +++++----------
src/bin/pg_waldump/pg_waldump.c | 21 +++++++++------------
src/include/access/xlogreader.h | 14 +++++---------
3 files changed, 19 insertions(+), 31 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ce901c358a..0bf8dac408 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1026,7 +1026,8 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
@@ -1065,9 +1066,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* Read the page containing the record */
while(XLogNeedData(state, targetPagePtr, targetRecOff))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
if (state->readLen < 0)
goto err;
@@ -1078,9 +1077,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* make sure we have enough data for the page header */
while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
if (state->readLen < 0)
goto err;
@@ -1127,9 +1124,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
{
if (result == XLREAD_NEED_DATA)
{
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
continue;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 54717c9320..3125633327 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static void
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -1095,13 +1097,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1128,12 +1130,7 @@ main(int argc, char **argv)
while (XLogReadRecord(xlogreader_state,
first_record, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader_state->read_page(xlogreader_state,
- xlogreader_state->loadPagePtr,
- xlogreader_state->loadLen,
- xlogreader_state->currRecPtr,
- xlogreader_state->readBuf,
- &xlogreader_state->readPageTLI);
+ XLogDumpReadPage(xlogreader_state, (void *) &private);
if (!record)
{
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index bc0c642906..b4ace71a75 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -29,14 +29,6 @@
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf,
- TimeLineID *pageTLI);
-
typedef struct
{
/* Is this block ref in use? */
@@ -252,7 +244,11 @@ extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
extern void XLogReaderInvalReadState(XLogReaderState *state);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef void (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
--
2.16.3
From b6392972840935be4faaa09df103ee6f77a8b9a8 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 16:00:57 +0900
Subject: [PATCH 09/10] Make pg_rewind not use callback but call the function
directly
This patch replaces the call to the callback in pg_rewind with direct
call to the original function. Then invalidate the parameters callback
and private for XLogReaderAllocate.
---
src/bin/pg_rewind/parsexlog.c | 78 ++++++++++++-------------------------------
1 file changed, 22 insertions(+), 56 deletions(-)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index e26127206c..26446027ab 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,16 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *datadir;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static void SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI);
+static void SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -64,12 +56,8 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -77,12 +65,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
{
while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -124,24 +107,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -177,7 +151,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -193,10 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -207,12 +177,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -259,11 +224,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogreader callback function, to read a WAL page */
static void
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI)
+SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char*datadir, int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->loadPagePtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *pageTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -296,17 +262,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
- snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
+ snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", datadir, xlogfname);
xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
@@ -347,7 +313,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- *pageTLI = targetHistory[private->tliIndex].tli;
+ *pageTLI = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return;
--
2.16.3
From fd8bb17b85cd0a35e76a3cf29c114d765d05fbbd Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 16:15:18 +0900
Subject: [PATCH 10/10] Remove callback entry from XLogReaderState
This is the third (3/2?) step. Remove no-longer-useful members
read_page and private_data from XLogReaderState. Then change
XLogReaderAllocate not to take the parameters.
---
src/backend/access/transam/twophase.c | 2 +-
src/backend/access/transam/xlog.c | 4 ++--
src/backend/access/transam/xlogreader.c | 9 ++-------
src/backend/replication/logical/logical.c | 2 +-
src/bin/pg_rewind/parsexlog.c | 6 +++---
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 32 +------------------------------
7 files changed, 11 insertions(+), 46 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 23424886dc..ec68f770bd 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1386,7 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9f184eefbe..73b343dca6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1188,7 +1188,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size);
if (!debug_reader)
{
@@ -6327,7 +6327,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0bf8dac408..bf61bdd230 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -149,8 +149,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
- void *private_data)
+XLogReaderAllocate(int wal_segment_size)
{
XLogReaderState *state;
@@ -178,11 +177,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
}
state->wal_segment_size = wal_segment_size;
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
- /* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* All members are initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 767d73f476..17d1b7ae1d 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -172,7 +172,7 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ ctx->reader = XLogReaderAllocate(wal_segment_size);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 26446027ab..65420d0e4a 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -57,7 +57,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -109,7 +109,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -166,7 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 3125633327..cec0a401ee 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1097,7 +1097,7 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b4ace71a75..e83fc4da0e 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -98,40 +98,12 @@ struct XLogReaderState
*/
int wal_segment_size;
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set *pageTLI to the TLI of the file the page was
- * read from. It is currently used only for error reporting purposes, to
- * reconstruct the name of the WAL file where an error occurred.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
@@ -223,9 +195,7 @@ struct XLogReaderState
};
/* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
--
2.16.3
Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> wrote: > v3-0001 : Changed macrosas suggested. This attachment is missing, please send it too. -- Antonin Houska Web: https://www.cybertec-postgresql.com
Hi, On 2019-04-18 21:02:57 +0900, Kyotaro HORIGUCHI wrote: > Hello. As mentioned before [1], read_page callback in > XLogReaderState is a cause of headaches. Adding another > remote-controlling stuff to xlog readers makes things messier [2]. > > I refactored XLOG reading functions so that we don't need the > callback. In short, ReadRecrod now calls XLogPageRead directly > with the attached patch set. > > | while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) > | == XLREAD_NEED_DATA) > | XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); > > On the other hand, XLogReadRecord became a bit complex. The patch > makes XLogReadRecord a state machine. I'm not confident that the > additional complexity is worth doing. Anyway I'll gegister this > to the next CF. Just FYI, to me this doesn't clearly enough look like an improvement, for a change of this size. Greetings, Andres Freund
Hello. The patch gets disliked by my tool chain. Fixed the usage
of PG_USED_FOR_ASSERTS_ONLY and rebased to bd56cd75d2.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 1dabdce6993b73408b950cb8c348c4999178b9a0 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 10:22:49 +0900
Subject: [PATCH 01/10] Define macros to make XLogReadRecord a state machine
To minimize apparent impact on code, use some macros as syntax
sugar. This is a similar stuff with ExecInterpExpr but a bit
different. The most significant difference is that this stuff allows
some functions are leaved midst of their work then continue. Roughly
speaking this is used as the follows.
enum retval
some_func()
{
static .. internal_variables;
XLR_SWITCH(INITIAL_STATUS);
...
XLR_LEAVE(STATUS1, RETVAL_CONTINUE);
...
XLR_LEAVE(STATUS2, RETVAL_CONTINUE2);
...
XLR_SWITCH_END();
XLR_RETURN(RETVAL_FINISH);
}
The caller uses the function as follows:
while (some_func() != RETVAL_FINISH)
{
<do some work>;
}
---
src/backend/access/transam/xlogreader.c | 83 +++++++++++++++++++++++++++++++++
1 file changed, 83 insertions(+)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 41dae916b4..69d20e1f2f 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -29,6 +29,89 @@
#include "utils/memutils.h"
#endif
+/*
+ * Use computed-goto-based state dispatch when computed gotos are available.
+ * But use a separate symbol so that it's easy to adjust locally in this file
+ * for development and testing.
+ */
+#ifdef HAVE_COMPUTED_GOTO
+#define XLR_USE_COMPUTED_GOTO
+#endif /* HAVE_COMPUTED_GOTO */
+
+/*
+ * The state machine functions relies on static local variables. They cannot
+ * be reentered after non-local exit using ereport/elog for consistency. The
+ * assertion macros protect the functions from reenter after non-local exit.
+ */
+#ifdef USE_ASSERT_CHECKING
+#define XLR_REENT_PROTECT_ENTER() \
+ do { Assert(!__xlr_running); __xlr_running = true; } while (0)
+#define XLR_REENT_PROTECT_LEAVE()\
+ do { __xlr_running = false; } while (0)
+#else
+#define XLR_REENT_PROTECT_ENTER()
+#define XLR_REENT_PROTECT_LEAVE()
+#endif
+
+/*
+ * Macros for state dispatch.
+ *
+ * XLR_SWITCH - prologue code for state machine including switch itself.
+ * XLR_CASE - labels the implementation of named state.
+ * XLR_LEAVE - leave the function and return here at the next call.
+ * XLR_RETURN - return from the function and set state to initial state.
+ * XLR_END - just hides the closing brace if not in use.
+ */
+#if defined(XLR_USE_COMPUTED_GOTO)
+#define XLR_SWITCH(name) \
+ static bool __xlr_running PG_USED_FOR_ASSERTS_ONLY = false; \
+ static void *__xlr_init_state = &&name; \
+ static void *__xlr_state = &&name; \
+ do { \
+ XLR_REENT_PROTECT_ENTER(); \
+ goto *__xlr_state; \
+ XLR_CASE(name); \
+ } while (0)
+#define XLR_CASE(name) name:
+#define XLR_LEAVE(name, code) \
+ do { \
+ __xlr_state = (&&name); \
+ XLR_REENT_PROTECT_LEAVE(); \
+ return (code); \
+ XLR_CASE(name); \
+ } while (0)
+#define XLR_RETURN(code) \
+ do { \
+ __xlr_state = __xlr_init_state; \
+ XLR_REENT_PROTECT_LEAVE(); \
+ return (code); \
+ } while (0)
+#define XLR_SWITCH_END()
+#else /* !XLR_USE_COMPUTED_GOTO */
+#define XLR_SWITCH(name) \
+ static bool __xlr_running = false PG_USED_FOR_ASSERTS_ONLY; \
+ static int __xlr_init_state = name; \
+ static int __xlr_state = name; \
+ XLR_REENT_PROTECT_ENTER(); \
+ switch (__xlr_state) { \
+ XLR_CASE(name)
+#define XLR_CASE(name) case name:
+#define XLR_LEAVE(name, code) \
+ do { \
+ __xlr_state = (name); \
+ XLR_REENT_PROTECT_LEAVE(); \
+ return (code); \
+ XLR_CASE(name); \
+ } while (0)
+#define XLR_RETURN(code) \
+ do { \
+ __xlr_state = __xlr_init_state; \
+ XLR_REENT_PROTECT_LEAVE(); \
+ return (code); \
+ } while (0)
+#define XLR_SWITCH_END() }
+#endif /* XLR_USE_COMPUTED_GOTO */
+
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
--
2.16.3
From 6c36b3c428ac1160b4211710331c1cfee5f83ab9 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Wed, 17 Apr 2019 14:15:16 +0900
Subject: [PATCH 02/10] Make ReadPageInternal a state machine
This patch set aims to remove read_page call back from
XLogReaderState. This is done in two steps. This patch does the first
step. Makes ReadPageInternal, which currently calling read_page
callback, a state machine and move the caller sites to XLogReadRecord
and other direct callers of the callback.
---
src/backend/access/transam/xlog.c | 15 ++-
src/backend/access/transam/xlogreader.c | 180 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 8 +-
src/backend/replication/logical/logicalfuncs.c | 6 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 ++-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 39 ++++--
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 177 insertions(+), 110 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b6c9353cbd..f517dec16c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -11518,7 +11518,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static void
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11577,7 +11577,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
}
@@ -11672,7 +11673,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11686,8 +11688,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 69d20e1f2f..45d201fc23 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -118,8 +118,8 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen);
static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3);
static void ResetDecoder(XLogReaderState *state);
@@ -306,7 +306,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -358,15 +357,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -392,8 +393,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -470,14 +471,17 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(total_len - gotlen + SizeOfXLogShortPHD,
+ XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
- if (readOff < 0)
+ if (state->readLen < 0)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -506,20 +510,28 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
+ if (state->readLen < pageHeaderSize)
+ {
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ }
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
+ if (state->readLen < pageHeaderSize + len)
+ {
+ if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ }
memcpy(buffer, (char *) contdata, len);
buffer += len;
@@ -550,9 +562,13 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ)))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/* Record does not cross a page boundary */
@@ -595,39 +611,48 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the required data is fully loaded. state->readLen is set to
+ * -1 when the loaded data is found to be invalid.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise, returns true and requests data using state->loadPagePtr and
+ * state->loadLen. The caller should load the region to state->readBuf and
+ * call this function again.
+ *
+ * Note: This function is not reentrant. The state is maintained internally in
+ * the function. DO NOT ereport(ERROR)-out from inside of this function.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
{
- int readLen;
- uint32 targetPageOff;
- XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ /*
+ * This function is a state machine that can exit and reenter at any place
+ * marked as XLR_LEAVE. All internal state is preserved through multiple
+ * calls.
+ */
+ static uint32 targetPageOff;
+ static XLogSegNo targetSegNo;
+ static XLogPageHeader hdr;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ XLR_SWITCH (XLND_STATE_INIT);
XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
/* check whether we have all the requested data already */
if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
reqLen <= state->readLen)
- return state->readLen;
+ XLR_RETURN(false);
/*
* Data is not in our buffer.
*
* Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
+ * before, we need to do verification as the caller might now be rereading
+ * data from a different source.
*
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
@@ -636,18 +661,17 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ state->loadPagePtr = pageptr - targetPageOff;
+ state->loadLen = XLOG_BLCKSZ;
+ XLR_LEAVE(XLND_STATE_SEGHEADER, true);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ if (state->readLen < 0)
goto err;
/* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ Assert(state->readLen == XLOG_BLCKSZ);
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
+ if (!XLogReaderValidatePageHeader(state, state->loadPagePtr,
state->readBuf))
goto err;
}
@@ -656,48 +680,53 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
* First, read the requested data length, but at least a short page header
* so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ state->loadPagePtr = pageptr;
+ state->loadLen = Max(reqLen, SizeOfXLogShortPHD);
+ XLR_LEAVE(XLND_STATE_PAGEHEADER, true);
+
+ if (state->readLen < 0)
goto err;
- Assert(readLen <= XLOG_BLCKSZ);
+ Assert(state->readLen <= XLOG_BLCKSZ);
/* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
+ if (state->readLen <= SizeOfXLogShortPHD)
goto err;
- Assert(readLen >= reqLen);
+ Assert(state->readLen >= state->loadLen);
hdr = (XLogPageHeader) state->readBuf;
/* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
+ if (state->readLen < XLogPageHeaderSize(hdr))
{
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
+ state->loadPagePtr = pageptr;
+ state->loadLen = XLogPageHeaderSize(hdr);
+ XLR_LEAVE(XLND_STATE_PAGELONGHEADER, true);
+
+ if (state->readLen < 0)
goto err;
}
+ XLR_SWITCH_END();
+
/*
* Now that we know we have the full header, validate it.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
+ if (!XLogReaderValidatePageHeader(state, pageptr, (char *) state->readBuf))
+ goto err;
/* update read state information */
state->readSegNo = targetSegNo;
state->readOff = targetPageOff;
- state->readLen = readLen;
- return readLen;
+ XLR_RETURN(false);
err:
XLogReaderInvalReadState(state);
- return -1;
+ state->readLen = -1;
+
+ XLR_RETURN(false);
}
/*
@@ -986,7 +1015,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -994,7 +1022,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -1003,8 +1031,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
targetPagePtr = tmpRecPtr - targetRecOff;
/* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
header = (XLogPageHeader) state->readBuf;
@@ -1012,8 +1044,12 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
pageHeaderSize = XLogPageHeaderSize(header);
/* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+
+ if (state->readLen < 0)
goto err;
/* skip over potential continuation data */
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 10a663bae6..c853e1f0e3 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+void
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..16c6095179 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+void
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ read_local_xlog_page(state, targetPagePtr, reqLen,
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e7a59b0a92..130656e5b1 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static void
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 287af60c4e..0bfbea8f09 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static void SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogreader callback function, to read a WAL page */
-static int
+static void
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -291,7 +291,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
}
@@ -304,7 +305,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
@@ -317,13 +319,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b95d467805..f4dfdd12ca 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static void
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 04228e2a87..d3b3e4b7ba 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -30,7 +30,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -66,6 +66,15 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* internal state of XLogNeedData() */
+typedef enum xlnd_stateid
+{
+ XLND_STATE_INIT,
+ XLND_STATE_SEGHEADER,
+ XLND_STATE_PAGEHEADER,
+ XLND_STATE_PAGELONGHEADER
+} xlnd_stateid;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -120,6 +129,22 @@ struct XLogReaderState
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* parameters to page reader */
+ XLogRecPtr loadPagePtr; /* Pointer to the page */
+ int loadLen; /* wanted length in bytes */
+ char *readBuf; /* buffer to store data */
+ XLogRecPtr currRecPtr; /* beginning of the WAL record being read */
+
+ /* return from page reader */
+ int32 readLen; /* bytes actually read, must be at least
+ * loadLen. -1 on error. */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+
/* ----------------------------------------
* Decoded representation of current record
*
@@ -145,17 +170,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
XLogSegNo readSegNo;
uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
@@ -164,8 +181,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..be0c79d18c 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern void read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..b0306c54ab 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern void logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
From 949d47fe75f33a2da7cbd44c6fd3e7116381e0a6 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Fri, 24 May 2019 09:22:44 +0900
Subject: [PATCH 03/10] Change interface of XLogReadRecord
As a preparation to the second step, this patch changes the interface
of XLogReadRecord so that the function can request the callers for
reading more data.
---
src/backend/access/transam/twophase.c | 2 +-
src/backend/access/transam/xlog.c | 2 +-
src/backend/access/transam/xlogreader.c | 52 ++++++++++++++------------
src/backend/replication/logical/logical.c | 2 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/slotfuncs.c | 2 +-
src/backend/replication/walsender.c | 2 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 13 ++++++-
10 files changed, 49 insertions(+), 36 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 477709bbc2..653f685870 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1394,7 +1394,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ XLogReadRecord(xlogreader, lsn, &record, &errormsg);
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f517dec16c..27ab6cc815 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4261,7 +4261,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
{
char *errormsg;
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
+ XLogReadRecord(xlogreader, RecPtr, &record, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 45d201fc23..8bd0e2925d 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -285,20 +285,21 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
* If RecPtr is valid, try to read a record at that position. Otherwise
* try to read a record just after the last one previously read.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * If the read_page callback fails to read the requested data, *record is set
+ * to NULL and XLREAD_FAIL is returned. The callback is expected to have
+ * reported the error; errormsg is set to NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * If the reading fails for some other reason, *record is also set to NULL and
+ * XLREAD_FAIL is returned. *errormsg is set to a string with details of the
+ * failure.
*
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogRecord **record, char **errormsg)
{
- XLogRecord *record;
XLogRecPtr targetPagePtr;
bool randAccess;
uint32 len,
@@ -405,8 +406,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* cannot access any other fields until we've verified that we got the
* whole header.
*/
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ *record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
+ total_len = (*record)->xl_tot_len;
/*
* If the whole record header is on this page, validate it immediately.
@@ -418,7 +419,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
*/
if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
{
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
+ if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, *record,
randAccess))
goto err;
gotheader = true;
@@ -540,9 +541,9 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* If we just reassembled the record header, validate it. */
if (!gotheader)
{
- record = (XLogRecord *) state->readRecordBuf;
+ *record = (XLogRecord *) state->readRecordBuf;
if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ *record, randAccess))
goto err;
gotheader = true;
}
@@ -550,8 +551,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
+ *record = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, *record, RecPtr))
goto err;
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
@@ -572,7 +573,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
/* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
+ if (!ValidXLogRecord(state, *record, RecPtr))
goto err;
state->EndRecPtr = RecPtr + MAXALIGN(total_len);
@@ -583,18 +584,19 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->wal_segment_size - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
@@ -607,7 +609,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -1001,6 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1089,7 +1093,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9853be6d1c..4f383721eb 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -481,7 +481,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ XLogReadRecord(ctx->reader, startptr, &record, &err);
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 16c6095179..6fc78d7445 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -289,7 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ XLogReadRecord(ctx->reader, startptr, &record, &errm);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 808a6f5b83..7db8e0d044 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ XLogReadRecord(ctx->reader, startlsn, &record, &errm);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 130656e5b1..4ba43592ca 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2827,7 +2827,7 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm);
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0bfbea8f09..512005de1c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -75,7 +75,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ XLogReadRecord(xlogreader, startpoint, &record, &errormsg);
if (record == NULL)
{
@@ -127,7 +127,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ XLogReadRecord(xlogreader, ptr, &record, &errormsg);
if (record == NULL)
{
if (errormsg)
@@ -190,7 +190,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ XLogReadRecord(xlogreader, searchptr, &record, &errormsg);
if (record == NULL)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f4dfdd12ca..41aa108215 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1132,7 +1132,7 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ XLogReadRecord(xlogreader_state, first_record, &record, &errormsg);
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index d3b3e4b7ba..0e734d27f1 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -66,6 +66,13 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
/* internal state of XLogNeedData() */
typedef enum xlnd_stateid
{
@@ -220,8 +227,10 @@ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
extern void XLogReaderFree(XLogReaderState *state);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
--
2.16.3
From c3ac6cb8c4d4478053686d9485b74e912949c391 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Fri, 24 May 2019 09:33:29 +0900
Subject: [PATCH 04/10] Make XLogReadRecord a state machine
This patch moves the caller sites of the callback above XLogReadRecord
by making XLogReadRecord a state machine.
---
src/backend/access/transam/twophase.c | 8 +-
src/backend/access/transam/xlog.c | 8 +-
src/backend/access/transam/xlogreader.c | 108 ++++++++++++++++---------
src/backend/replication/logical/logical.c | 10 ++-
src/backend/replication/logical/logicalfuncs.c | 10 ++-
src/backend/replication/slotfuncs.c | 10 ++-
src/backend/replication/walsender.c | 10 ++-
src/bin/pg_rewind/parsexlog.c | 28 ++++++-
src/bin/pg_waldump/pg_waldump.c | 11 ++-
src/include/access/xlogreader.h | 12 +++
10 files changed, 165 insertions(+), 50 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 653f685870..6feca69126 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1394,7 +1394,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- XLogReadRecord(xlogreader, lsn, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr, xlogreader->loadLen,
+ xlogreader->currRecPtr, xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 27ab6cc815..573b49050d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4261,7 +4261,13 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
{
char *errormsg;
- XLogReadRecord(xlogreader, RecPtr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg)
+ == XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr, xlogreader->loadLen,
+ xlogreader->currRecPtr, xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 8bd0e2925d..4ab0655af5 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -282,31 +282,57 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * This function runs a state machine and may need to call several times until
+ * a record is read.
*
- * If the read_page callback fails to read the requested data, *record is set
- * to NULL and XLREAD_FAIL is returned. The callback is expected to have
- * reported the error; errormsg is set to NULL.
+ * At the initial state, if called with valid pRecPtr, try to read a record at
+ * that position. If invalid pRecPtr is given try to read a record just after
+ * the last one previously read.
*
- * If the reading fails for some other reason, *record is also set to NULL and
- * XLREAD_FAIL is returned. *errormsg is set to a string with details of the
- * failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record then the state machine is reset to initial
+ * state.
+ *
+ * Returns XLREAD_NEED_DATA if needs more data fed. In that case loadPagePtr
+ * and loadLen in state is set to inform the required WAL data. The caller
+ * shall read in the requested data into readBuf and set readLen and
+ * readPageTLI to the length of the data actually read and the TLI for the
+ * data read in respectively. In case of failure the caller shall call the
+ * function setting readLen to -1 and storing error message in errormsg_buf to
+ * inform error.
+ *
+ * If the reading fails for some reasons including caller-side error mentioned
+ * above, returns XLREAD_FAIL with *record being set to NULL. *errormsg is set
+ * to a string with details of the failure. The state machine is reset to
+ * initial state.
*
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
+ *
+ * Note: This function is not reentrant. The state is maintained internally in
+ * the function. DO NOT non-local exit (ereport) from inside of this function.
*/
XLogReadRecordResult
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+XLogReadRecord(XLogReaderState *state, XLogRecPtr pRecPtr,
XLogRecord **record, char **errormsg)
{
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ /*
+ * This function is a state machine that can exit and reenter at any place
+ * marked as XLR_LEAVE. All internal state needs to be preserved through
+ * multiple calls.
+ */
+ static XLogRecPtr targetPagePtr;
+ static bool randAccess;
+ static uint32 len,
+ total_len;
+ static uint32 targetRecOff;
+ static uint32 pageHeaderSize;
+ static bool gotheader;
+ static XLogRecPtr RecPtr;
+
+ XLR_SWITCH(XLREAD_STATE_INIT);
+
+ RecPtr = pRecPtr;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -360,10 +386,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*/
while (XLogNeedData(state, targetPagePtr,
Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
-
+ XLR_LEAVE(XLREAD_STATE_PAGE, XLREAD_NEED_DATA);
+
if (state->readLen < 0)
goto err;
@@ -442,10 +466,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (total_len > len)
{
/* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ static char *contdata;
+ static XLogPageHeader pageHeader;
+ static char *buffer;
+ static uint32 gotlen;
/*
* Enlarge readRecordBuf as needed.
@@ -475,9 +499,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
while (XLogNeedData(state, targetPagePtr,
Min(total_len - gotlen + SizeOfXLogShortPHD,
XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTPAGE, XLREAD_NEED_DATA);
if (state->readLen < 0)
goto err;
@@ -514,9 +536,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (state->readLen < pageHeaderSize)
{
while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTPAGE_HEADER, XLREAD_NEED_DATA);
}
Assert(pageHeaderSize <= state->readLen);
@@ -529,9 +549,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
if (state->readLen < pageHeaderSize + len)
{
if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_CONTRECORD, XLREAD_NEED_DATA);
}
memcpy(buffer, (char *) contdata, len);
@@ -565,9 +583,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
/* Wait for the record data to become available */
while (XLogNeedData(state, targetPagePtr,
Min(targetRecOff + total_len, XLOG_BLCKSZ)))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ XLR_LEAVE(XLREAD_STATE_RECORD, XLREAD_NEED_DATA);
if (state->readLen < 0)
goto err;
@@ -581,6 +597,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
state->ReadRecPtr = RecPtr;
}
+ XLR_SWITCH_END();
+
/*
* Special processing if it's an XLOG SWITCH record
*/
@@ -593,10 +611,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr,
}
if (DecodeXLogRecord(state, *record, errormsg))
- return XLREAD_SUCCESS;
+ XLR_RETURN(XLREAD_SUCCESS);
*record = NULL;
- return XLREAD_FAIL;
+ XLR_RETURN(XLREAD_FAIL);
err:
@@ -610,7 +628,7 @@ err:
*errormsg = state->errormsg_buf;
*record = NULL;
- return XLREAD_FAIL;
+ XLR_RETURN(XLREAD_FAIL);
}
/*
@@ -1005,6 +1023,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1093,8 +1112,17 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &record, &errormsg) == XLREAD_SUCCESS)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ state->read_page(state, state->loadPagePtr, state->loadLen,
+ state->currRecPtr, state->readBuf,
+ &state->readPageTLI);
+ continue;
+ }
+
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 4f383721eb..06200ea2e9 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -481,7 +481,15 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- XLogReadRecord(ctx->reader, startptr, &record, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 6fc78d7445..4d09255504 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -289,7 +289,15 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- XLogReadRecord(ctx->reader, startptr, &record, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 7db8e0d044..f4f4a907ad 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,15 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- XLogReadRecord(ctx->reader, startlsn, &record, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ ctx->reader->read_page(ctx->reader,
+ ctx->reader->loadPagePtr,
+ ctx->reader->loadLen,
+ ctx->reader->currRecPtr,
+ ctx->reader->readBuf,
+ &ctx->reader->readPageTLI);
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4ba43592ca..36e14ab822 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -2827,7 +2827,15 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &record, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader,
+ logical_decoding_ctx->reader->loadPagePtr,
+ logical_decoding_ctx->reader->loadLen,
+ logical_decoding_ctx->reader->currRecPtr,
+ logical_decoding_ctx->reader->readBuf,
+ &logical_decoding_ctx->reader->readPageTLI);
+
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 512005de1c..e26127206c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -75,7 +75,14 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
do
{
- XLogReadRecord(xlogreader, startpoint, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
if (record == NULL)
{
@@ -127,7 +134,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
if (xlogreader == NULL)
pg_fatal("out of memory");
- XLogReadRecord(xlogreader, ptr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
+
if (record == NULL)
{
if (errormsg)
@@ -190,7 +205,14 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- XLogReadRecord(xlogreader, searchptr, &record, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader->read_page(xlogreader,
+ xlogreader->loadPagePtr,
+ xlogreader->loadLen,
+ xlogreader->currRecPtr,
+ xlogreader->readBuf,
+ &xlogreader->readPageTLI);
if (record == NULL)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 41aa108215..e2e93f144a 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1132,7 +1132,16 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- XLogReadRecord(xlogreader_state, first_record, &record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ xlogreader_state->read_page(xlogreader_state,
+ xlogreader_state->loadPagePtr,
+ xlogreader_state->loadLen,
+ xlogreader_state->currRecPtr,
+ xlogreader_state->readBuf,
+ &xlogreader_state->readPageTLI);
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0e734d27f1..bc0c642906 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -70,6 +70,7 @@ typedef struct
typedef enum XLogReadRecordResult
{
XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
XLREAD_FAIL /* failed during reading a record */
} XLogReadRecordResult;
@@ -82,6 +83,17 @@ typedef enum xlnd_stateid
XLND_STATE_PAGELONGHEADER
} xlnd_stateid;
+/* internal state of XLogReadRecord() */
+typedef enum xlread_stateid
+{
+ XLREAD_STATE_INIT,
+ XLREAD_STATE_PAGE,
+ XLREAD_STATE_CONTPAGE,
+ XLREAD_STATE_CONTPAGE_HEADER,
+ XLREAD_STATE_CONTRECORD,
+ XLREAD_STATE_RECORD
+} xlread_stateid;
+
struct XLogReaderState
{
/* ----------------------------------------
--
2.16.3
From a6862ebfdc4a82b489c95ba40f89e680d26fac51 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 13:48:25 +0900
Subject: [PATCH 05/10] Make XLogPageRead not use callback but call the
function directly
This patch replaces the call to the callback in XLogReadRecord with
direct call to the original function. Then invalidate the parameters
callback and private for XLogReaderAllocate.
---
src/backend/access/transam/xlog.c | 44 +++++++++++++--------------------------
1 file changed, 14 insertions(+), 30 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 573b49050d..f7b2528f26 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,9 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static void XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *readTLI);
+static void XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -4247,12 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- /* Pass through parameters to XLogPageRead */
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4263,10 +4250,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg)
== XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr, xlogreader->loadLen,
- xlogreader->currRecPtr, xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
@@ -6214,7 +6198,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6355,8 +6338,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11525,12 +11507,14 @@ CancelBackup(void)
* sleep and retry.
*/
static void
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ XLogRecPtr targetPagePtr = xlogreader->loadPagePtr;
+ int reqLen = xlogreader->loadLen;
+ XLogRecPtr targetRecPtr = xlogreader->currRecPtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *readTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11573,8 +11557,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
--
2.16.3
From b233f84e3ce28b6634ae6f301088b54e6cfd0e4c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:02:19 +0900
Subject: [PATCH 06/10] Make XlogReadTwoPhaseData not use callback but call the
function directly
This patch replaces the call to the callback in XlogReadTwoPhaseData
with direct call to the original function. Then invalidate the
parameters callback and private for XLogReaderAllocate.
---
src/backend/access/transam/twophase.c | 8 ++------
src/backend/access/transam/xlogutils.c | 8 +++++---
src/backend/replication/logical/logicalfuncs.c | 8 ++++++--
src/include/access/xlogutils.h | 5 +----
4 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 6feca69126..f6d48368fe 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1386,8 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page,
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1396,10 +1395,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr, xlogreader->loadLen,
- xlogreader->currRecPtr, xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ read_local_xlog_page(xlogreader);
if (record == NULL)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index c853e1f0e3..fd461f16fc 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -908,10 +908,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
void
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *cur_page = state->readBuf;
+ TimeLineID *pageTLI = &state->readPageTLI;
XLogRecPtr read_upto,
loc;
int count;
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 4d09255504..240a375d8f 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -118,8 +118,12 @@ void
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
- read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ Assert(targetPagePtr == state->loadPagePtr &&
+ reqLen == state->loadLen &&
+ targetRecPtr == state->currRecPtr &&
+ cur_page == state->readBuf &&
+ pageTLI == &state->readPageTLI);
+ read_local_xlog_page(state);
}
/*
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index be0c79d18c..9724ce20b8 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern void read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI);
+extern void read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
--
2.16.3
From 0682f7f9472f443f77505ce17745f98acf7f5a7c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:35:10 +0900
Subject: [PATCH 07/10] Make logical rep stuff not use callback but call the
function directly
This is a bit different from the two before. This patch moves the
callback from XLogReaderState to LogicalDecodingContext. Then
invalidate the parameters callback and private for XLogReaderAllocate.
---
src/backend/replication/logical/logical.c | 17 +++++++----------
src/backend/replication/logical/logicalfuncs.c | 17 +++--------------
src/backend/replication/slotfuncs.c | 7 +------
src/backend/replication/walsender.c | 15 +++++++--------
src/include/replication/logical.h | 13 +++++++++----
src/include/replication/logicalfuncs.h | 5 +----
6 files changed, 28 insertions(+), 46 deletions(-)
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 06200ea2e9..dacd73e031 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -123,7 +123,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -172,12 +172,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
+
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
AllocateSnapshotBuilder(ctx->reorder, xmin_horizon, start_lsn,
@@ -231,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -373,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -483,12 +485,7 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
/* the read_page callback waits for new WAL */
while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (err)
elog(ERROR, "%s", err);
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 240a375d8f..2a3f6d3cde 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,15 +115,9 @@ check_permissions(void)
}
void
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- Assert(targetPagePtr == state->loadPagePtr &&
- reqLen == state->loadLen &&
- targetRecPtr == state->currRecPtr &&
- cur_page == state->readBuf &&
- pageTLI == &state->readPageTLI);
- read_local_xlog_page(state);
+ read_local_xlog_page(ctx->reader);
}
/*
@@ -295,12 +289,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f4f4a907ad..8675c9203e 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -435,12 +435,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
*/
while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
XLREAD_NEED_DATA)
- ctx->reader->read_page(ctx->reader,
- ctx->reader->loadPagePtr,
- ctx->reader->loadLen,
- ctx->reader->currRecPtr,
- ctx->reader->readBuf,
- &ctx->reader->readPageTLI);
+ ctx->read_page(ctx);
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 36e14ab822..5bf8646c80 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,9 +762,13 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static void
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *cur_page = state->readBuf;
+
XLogRecPtr flushptr;
int count;
@@ -2829,12 +2833,7 @@ XLogSendLogical(void)
while (XLogReadRecord(logical_decoding_ctx->reader,
logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
- logical_decoding_ctx->reader->read_page(logical_decoding_ctx->reader,
- logical_decoding_ctx->reader->loadPagePtr,
- logical_decoding_ctx->reader->loadLen,
- logical_decoding_ctx->reader->currRecPtr,
- logical_decoding_ctx->reader->readBuf,
- &logical_decoding_ctx->reader->readPageTLI);
+ logical_decoding_ctx->read_page(logical_decoding_ctx);
logical_startptr = InvalidXLogRecPtr;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index f76b9eb4df..f0bec63241 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -32,7 +32,11 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (
TransactionId xid
);
-typedef struct LogicalDecodingContext
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef void (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
+struct LogicalDecodingContext
{
/* memory context this is all allocated in */
MemoryContext context;
@@ -42,6 +46,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -89,7 +94,7 @@ typedef struct LogicalDecodingContext
bool prepared_write;
XLogRecPtr write_location;
TransactionId write_xid;
-} LogicalDecodingContext;
+};
extern void CheckLogicalDecodingRequirements(void);
@@ -98,7 +103,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
@@ -106,7 +111,7 @@ extern LogicalDecodingContext *CreateDecodingContext(
XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index b0306c54ab..04a9fe10fa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern void logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page, TimeLineID *pageTLI);
+extern void logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From 1f1ca75055bb17a1e9e877527f2094f12ac41ca6 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 15:50:51 +0900
Subject: [PATCH 08/10] Make pg_waldump not use callback but call the function
directly
This patch does the similar thing to the change in logical rep. Moves
callback from XLogReaderState from the parameter of
XLogFindNextRecord. Then invalidate the parameters callback and
private for XLogReaderAllocate.
---
src/backend/access/transam/xlogreader.c | 15 +++++----------
src/bin/pg_waldump/pg_waldump.c | 21 +++++++++------------
src/include/access/xlogreader.h | 14 +++++---------
3 files changed, 19 insertions(+), 31 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 4ab0655af5..004eaac021 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1016,7 +1016,8 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
@@ -1055,9 +1056,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* Read the page containing the record */
while(XLogNeedData(state, targetPagePtr, targetRecOff))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
if (state->readLen < 0)
goto err;
@@ -1068,9 +1067,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* make sure we have enough data for the page header */
while (XLogNeedData(state, targetPagePtr, pageHeaderSize))
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
if (state->readLen < 0)
goto err;
@@ -1117,9 +1114,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
{
if (result == XLREAD_NEED_DATA)
{
- state->read_page(state, state->loadPagePtr, state->loadLen,
- state->currRecPtr, state->readBuf,
- &state->readPageTLI);
+ read_page(state, private);
continue;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index e2e93f144a..8fe6823b32 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static void
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->loadPagePtr;
+ int reqLen = state->loadLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -1102,13 +1104,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1135,12 +1137,7 @@ main(int argc, char **argv)
while (XLogReadRecord(xlogreader_state,
first_record, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader_state->read_page(xlogreader_state,
- xlogreader_state->loadPagePtr,
- xlogreader_state->loadLen,
- xlogreader_state->currRecPtr,
- xlogreader_state->readBuf,
- &xlogreader_state->readPageTLI);
+ XLogDumpReadPage(xlogreader_state, (void *) &private);
if (!record)
{
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index bc0c642906..b4ace71a75 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -29,14 +29,6 @@
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef void (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf,
- TimeLineID *pageTLI);
-
typedef struct
{
/* Is this block ref in use? */
@@ -252,7 +244,11 @@ extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
extern void XLogReaderInvalReadState(XLogReaderState *state);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef void (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
--
2.16.3
From 598c7adabcb9c0063e74632ae4486c83c4ff474e Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 16:00:57 +0900
Subject: [PATCH 09/10] Make pg_rewind not use callback but call the function
directly
This patch replaces the call to the callback in pg_rewind with direct
call to the original function. Then invalidate the parameters callback
and private for XLogReaderAllocate.
---
src/bin/pg_rewind/parsexlog.c | 78 ++++++++++++-------------------------------
1 file changed, 22 insertions(+), 56 deletions(-)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index e26127206c..26446027ab 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,16 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *datadir;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static void SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI);
+static void SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -64,12 +56,8 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -77,12 +65,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
{
while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -124,24 +107,15 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -177,7 +151,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -193,10 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -207,12 +177,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
- xlogreader->read_page(xlogreader,
- xlogreader->loadPagePtr,
- xlogreader->loadLen,
- xlogreader->currRecPtr,
- xlogreader->readBuf,
- &xlogreader->readPageTLI);
+ SimpleXLogPageRead(xlogreader, datadir, &tliIndex);
if (record == NULL)
{
@@ -259,11 +224,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogreader callback function, to read a WAL page */
static void
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI)
+SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char*datadir, int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->loadPagePtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *pageTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -296,17 +262,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
- snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
+ snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", datadir, xlogfname);
xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
@@ -347,7 +313,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- *pageTLI = targetHistory[private->tliIndex].tli;
+ *pageTLI = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return;
--
2.16.3
From 38ceaa8ada19b5fd34af14500d2f2307489b75da Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 18 Apr 2019 16:15:18 +0900
Subject: [PATCH 10/10] Remove callback entry from XLogReaderState
This is the third (3/2?) step. Remove no-longer-useful members
read_page and private_data from XLogReaderState. Then change
XLogReaderAllocate not to take the parameters.
---
src/backend/access/transam/twophase.c | 2 +-
src/backend/access/transam/xlog.c | 4 ++--
src/backend/access/transam/xlogreader.c | 9 ++-------
src/backend/replication/logical/logical.c | 2 +-
src/bin/pg_rewind/parsexlog.c | 6 +++---
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 32 +------------------------------
7 files changed, 11 insertions(+), 46 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f6d48368fe..e7a086b71e 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1386,7 +1386,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f7b2528f26..546ea09996 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1187,7 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size);
if (!debug_reader)
{
@@ -6338,7 +6338,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 004eaac021..443a7b608f 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -149,8 +149,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
- void *private_data)
+XLogReaderAllocate(int wal_segment_size)
{
XLogReaderState *state;
@@ -178,11 +177,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
}
state->wal_segment_size = wal_segment_size;
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
- /* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* All members are initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index dacd73e031..3724cff0fc 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -172,7 +172,7 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ ctx->reader = XLogReaderAllocate(wal_segment_size);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 26446027ab..65420d0e4a 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -57,7 +57,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -109,7 +109,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -166,7 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 8fe6823b32..54fff797fc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1104,7 +1104,7 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, NULL, NULL);
+ xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b4ace71a75..e83fc4da0e 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -98,40 +98,12 @@ struct XLogReaderState
*/
int wal_segment_size;
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set *pageTLI to the TLI of the file the page was
- * read from. It is currently used only for error reporting purposes, to
- * reconstruct the name of the WAL file where an error occurred.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
@@ -223,9 +195,7 @@ struct XLogReaderState
};
/* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
--
2.16.3
At Tue, 28 May 2019 04:45:24 -0700, Andres Freund <andres@anarazel.de> wrote in <20190528114524.dvj6ymap2virlzro@alap3.anarazel.de> > Hi, > > On 2019-04-18 21:02:57 +0900, Kyotaro HORIGUCHI wrote: > > Hello. As mentioned before [1], read_page callback in > > XLogReaderState is a cause of headaches. Adding another > > remote-controlling stuff to xlog readers makes things messier [2]. > > > > I refactored XLOG reading functions so that we don't need the > > callback. In short, ReadRecrod now calls XLogPageRead directly > > with the attached patch set. > > > > | while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) > > | == XLREAD_NEED_DATA) > > | XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); > > > > On the other hand, XLogReadRecord became a bit complex. The patch > > makes XLogReadRecord a state machine. I'm not confident that the > > additional complexity is worth doing. Anyway I'll gegister this > > to the next CF. > > Just FYI, to me this doesn't clearly enough look like an improvement, > for a change of this size. Thanks for the opiniton. I kinda agree about size but it is a decision between "having multiple callbacks called under the hood" vs "just calling a series of functions". I think the patched XlogReadRecord is easy to use in many situations. It would be better if I could completely refactor the function without the syntax tricks but I think the current patch is still smaller and clearer than overhauling it. If many of the folks think that adding a callback is better than this refactoring, I will withdraw this.. reagrds. -- Kyotaro Horiguchi NTT Open Source Software Center
On 12/07/2019 10:10, Kyotaro Horiguchi wrote: > At Tue, 28 May 2019 04:45:24 -0700, Andres Freund <andres@anarazel.de> wrote in <20190528114524.dvj6ymap2virlzro@alap3.anarazel.de> >> Hi, >> >> On 2019-04-18 21:02:57 +0900, Kyotaro HORIGUCHI wrote: >>> Hello. As mentioned before [1], read_page callback in >>> XLogReaderState is a cause of headaches. Adding another >>> remote-controlling stuff to xlog readers makes things messier [2]. >>> >>> I refactored XLOG reading functions so that we don't need the >>> callback. In short, ReadRecrod now calls XLogPageRead directly >>> with the attached patch set. >>> >>> | while (XLogReadRecord(xlogreader, RecPtr, &record, &errormsg) >>> | == XLREAD_NEED_DATA) >>> | XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess); >>> >>> On the other hand, XLogReadRecord became a bit complex. The patch >>> makes XLogReadRecord a state machine. I'm not confident that the >>> additional complexity is worth doing. Anyway I'll gegister this >>> to the next CF. >> >> Just FYI, to me this doesn't clearly enough look like an improvement, >> for a change of this size. > > Thanks for the opiniton. I kinda agree about size but it is a > decision between "having multiple callbacks called under the > hood" vs "just calling a series of functions". I think the > patched XlogReadRecord is easy to use in many situations. > > It would be better if I could completely refactor the function > without the syntax tricks but I think the current patch is still > smaller and clearer than overhauling it. I like the idea of refactoring XLogReadRecord() to not use a callback, and return a XLREAD_NEED_DATA value instead. It feels like a nicer, easier-to-use, interface, given that all the page-read functions need quite a bit of state and internal logic themselves. I remember that I felt that that would be a nicer interface when we originally extracted xlogreader.c into a reusable module, but I didn't want to make such big changes to XLogReadRecord() at that point. I don't much like the "continuation" style of implementing the state machine. Nothing wrong with such a style in principle, but we don't do that anywhere else, and the macros seem like overkill, and turning the local variables static is pretty ugly. But I think XLogReadRecord() could be rewritten into a more traditional state machine. I started hacking on that, to get an idea of what it would look like and came up with the attached patch, to be applied on top of all your patches. It's still very messy, it needs quite a lot of cleanup before it can be committed, but I think the resulting switch-case state machine in XLogReadRecord() is quite straightforward at high level, with four states. I made some further changes to the XLogReadRecord() interface: * If you pass a valid ReadPtr (i.e. the starting point to read from) argument to XLogReadRecord(), it always restarts reading from that record, even if it was in the middle of reading another record previously. (Perhaps it would be more convenient to provide a separate function to set the starting point, and remove the RecPtr argument from XLogReadRecord altogther?) * XLogReaderState->readBuf is now allocated and controlled by the caller, not by xlogreader.c itself. When XLogReadRecord() needs data, the caller makes the data available in readBuf, which can point to the same buffer in all calls, or the caller may allocate a new buffer, or it may point to a part of a larger buffer, whatever is convenient for the caller. (Currently, all callers just allocate a BLCKSZ'd buffer, though). The caller also sets readPagPtr, readLen and readPageTLI to tell XLogReadRecord() what's in the buffer. So all these read* fields are now set by the caller, XLogReadRecord() only reads them. * In your patch, if XLogReadRecord() was called with state->readLen == -1, XLogReadRecord() returned an error. That seemed a bit silly; if an error happened while reading the data, why call XLogReadRecord() at all? You could just report the error directly. So I removed that. I'm not sure how intelligible this patch is in its current state. But I think the general idea is good. I plan to clean it up further next week, but feel free to work on it before that, either based on this patch or by starting afresh from your patch set. - Heikki
Вложения
Thank you for the suggestion, Heikki. At Mon, 29 Jul 2019 22:39:57 +0300, Heikki Linnakangas <hlinnaka@iki.fi> wrote in <e1ecb53b-663d-98ed-2249-dfa30a74f8c1@iki.fi> > On 12/07/2019 10:10, Kyotaro Horiguchi wrote: > >> Just FYI, to me this doesn't clearly enough look like an improvement, > >> for a change of this size. > > Thanks for the opiniton. I kinda agree about size but it is a > > decision between "having multiple callbacks called under the > > hood" vs "just calling a series of functions". I think the > > patched XlogReadRecord is easy to use in many situations. > > It would be better if I could completely refactor the function > > without the syntax tricks but I think the current patch is still > > smaller and clearer than overhauling it. > > I like the idea of refactoring XLogReadRecord() to not use a callback, > and return a XLREAD_NEED_DATA value instead. It feels like a nicer, > easier-to-use, interface, given that all the page-read functions need > quite a bit of state and internal logic themselves. I remember that I > felt that that would be a nicer interface when we originally extracted > xlogreader.c into a reusable module, but I didn't want to make such > big changes to XLogReadRecord() at that point. > > I don't much like the "continuation" style of implementing the state > machine. Nothing wrong with such a style in principle, but we don't do > that anywhere else, and the macros seem like overkill, and turning the Agreed that it's a kind of ugly. I could overhaul the logic to reduce state variables, but I thought that it would make the patch hardly reviewable. The "continuation" style was intended to impact the main path's shape as small as possible. For the same reason I made variables static instead of using individual state struct or reducing state variables. (And it the style was fun for me:p) > local variables static is pretty ugly. But I think XLogReadRecord() > could be rewritten into a more traditional state machine. > > I started hacking on that, to get an idea of what it would look like > and came up with the attached patch, to be applied on top of all your > patches. It's still very messy, it needs quite a lot of cleanup before > it can be committed, but I think the resulting switch-case state > machine in XLogReadRecord() is quite straightforward at high level, > with four states. Sorry for late reply. It seems less messy than I thought it could be if I refactored it more aggressively. > I made some further changes to the XLogReadRecord() interface: > > * If you pass a valid ReadPtr (i.e. the starting point to read from) > * argument to XLogReadRecord(), it always restarts reading from that > * record, even if it was in the middle of reading another record > * previously. (Perhaps it would be more convenient to provide a separate > * function to set the starting point, and remove the RecPtr argument > * from XLogReadRecord altogther?) Seems reasonable. randAccess property was replaced with the state.PrevRecPtr = Invalid. It is easier to understand for me. > * XLogReaderState->readBuf is now allocated and controlled by the > * caller, not by xlogreader.c itself. When XLogReadRecord() needs data, > * the caller makes the data available in readBuf, which can point to the > * same buffer in all calls, or the caller may allocate a new buffer, or > * it may point to a part of a larger buffer, whatever is convenient for > * the caller. (Currently, all callers just allocate a BLCKSZ'd buffer, > * though). The caller also sets readPagPtr, readLen and readPageTLI to > * tell XLogReadRecord() what's in the buffer. So all these read* fields > * are now set by the caller, XLogReadRecord() only reads them. The caller knows how many byes to be read. So the caller provides the required buffer seems reasonable. > * In your patch, if XLogReadRecord() was called with state->readLen == > * -1, XLogReadRecord() returned an error. That seemed a bit silly; if an > * error happened while reading the data, why call XLogReadRecord() at > * all? You could just report the error directly. So I removed that. Agreed. I forgot to move the error handling to more proper location. > I'm not sure how intelligible this patch is in its current state. But > I think the general idea is good. I plan to clean it up further next > week, but feel free to work on it before that, either based on this > patch or by starting afresh from your patch set. I think you diff is intelligible enough for me. I'll take this if you haven't done. Anyway I'm staring on this. Thanks! -- Kyotaro Horiguchi NTT Open Source Software Center
On 22/08/2019 04:43, Kyotaro Horiguchi wrote: > At Mon, 29 Jul 2019 22:39:57 +0300, Heikki Linnakangas <hlinnaka@iki.fi> wrote in <e1ecb53b-663d-98ed-2249-dfa30a74f8c1@iki.fi> >> On 12/07/2019 10:10, Kyotaro Horiguchi wrote: >> * XLogReaderState->readBuf is now allocated and controlled by the >> * caller, not by xlogreader.c itself. When XLogReadRecord() needs data, >> * the caller makes the data available in readBuf, which can point to the >> * same buffer in all calls, or the caller may allocate a new buffer, or >> * it may point to a part of a larger buffer, whatever is convenient for >> * the caller. (Currently, all callers just allocate a BLCKSZ'd buffer, >> * though). The caller also sets readPagPtr, readLen and readPageTLI to >> * tell XLogReadRecord() what's in the buffer. So all these read* fields >> * are now set by the caller, XLogReadRecord() only reads them. > > The caller knows how many byes to be read. So the caller provides > the required buffer seems reasonable. I also had in mind that the caller could provide a larger buffer, spanning multiple pages, in one XLogReadRecord() call. It might be convenient to load a whole WAL file in memory and pass it to XLogReadRecord() in one call, for example. I think the interface would now allow that, but the code won't actually take advantage of that. XLogReadRecord() will always ask for one page at a time, and I think it will ask the caller for more data between each page, even if the caller supplies more than one page in one call. >> I'm not sure how intelligible this patch is in its current state. But >> I think the general idea is good. I plan to clean it up further next >> week, but feel free to work on it before that, either based on this >> patch or by starting afresh from your patch set. > > I think you diff is intelligible enough for me. I'll take this if > you haven't done. Anyway I'm staring on this. Thanks! I did actually spend some time on this last week, but got distracted by something else before finishing it up and posting a patch. Here's a snapshot of what I have in my local branch. It seems to pass "make check-world", but probably needs some more cleanup. Main changes since last version: * I changed the interface so that there is a new function to set the starting position, XLogBeginRead(), and XLogReadRecord() always continues from where it left off. I think that's more clear than having a starting point argument in XLogReadRecord(), which was only set on the first call. It makes the calling code more clear, too, IMO. * Refactored the implementation of XLogFindNextRecord(). XLogFindNextRecord() is now a sibling function of XLogBeginRead(). It sets the starting point like XLogBeginRead(). The difference is that with XLogFindNextRecord(), the starting point doesn't need to point to a valid record, it will "fast forward" to the next valid record after the point. The "fast forwarding" is done in an extra state in the state machine in XLogReadRecord(). * I refactored XLogReadRecord() and the internal XLogNeedData() function. XLogNeedData() used to contain logic for verifying segment and page headers. That works quite differently now. Checking the segment header is now a new state in the state machine, and the page header is verified at the top of XLogReadRecord(), whenever the caller provides new data. I think that makes the code in XLogReadRecord() more clear. - Heikki
Вложения
At Thu, 22 Aug 2019 10:43:52 +0900 (Tokyo Standard Time), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
<20190822.104352.26342272.horikyota.ntt@gmail.com>
> I think you diff is intelligible enough for me. I'll take this if
> you haven't done. Anyway I'm staring on this.
- Reducing state variables
It was a problem for me that there seems to be many state
variables than required. So first I tried to reduce them.
Now readPagePtr and readLen are used bidirectionally.
XLogNeedData sets it as request and page reader set readLen to
the actual length. Similarly verified* changes only when page
header is verified, so I introduced page_verified instead of the
variables.
- Changed calling convention of XLogReadRecord
To make caller loop simple, XLogReadRecord now allows to specify
the same valid value while reading the a record. No longer need
to change lsn to invalid after the first call in the following
reader loop.
while (XLogReadRecord(state, lsn, &record, &errormsg) == XLREAD_NEED_DATA)
{
if (!page_reader(state))
break;
}
- Frequent data request caused by seeing long page header.
XLogNeedData now takes the fourth parameter includes_page_header.
True means the caller is requesting with reqLen that is not
counting page header length. But it makes the function a bit too
complex than expected. Blindly requsting anticipating long page
header for a new page may prevent page-reader from returning the
bytes already at hand by waiting for bytes that won't come. To
prevent such a case the funtion should request anticipating short
page header first for a new page, then make a re-request using
SizeOfLongPHD if needed. Of course it is unlikely to happen for
file sources, and unlikely to harm physical replication (and the
behavior is not changed). Finally, the outcome is more or less
the same with just stashing the seemingly bogus retry from
XLogReadRecord to XLogNeedData. If we are allowed to utilize the
knowlege that long page header is attached to only the first page
of a segment, such complexitly could be eliminated.
- Moving page buffer allocation
As for page buffer allocation, I'm not sure it is meaningful, as
the reader assumes the buffer is in the same with page size,
which is immutable system-wide. It would be surely meanintful if
it were on the caller to decide its own block size, or loading
unit. Anyway it is in the third patch.
- Restored early check-out of record header
The current record reader code seems to be designed to bail-out
by broken record header as earlier as possible, perhaps in order
to prevent impossible size of read in. So I restored the
behavior.
The attched are the current status, it is separated to two
significant parts plus one for readability.
v5-0001-Move-callback-call-from-ReadPageInternal-to-XLogR.patch:
ReadPageInternal part of the patch. Moves callback calls from
ReadPageInternal up to XLogReadRecord. Some of recovery tests
fail applyin only this one but I don't want to put more efforts
to make this state perfecgt.
v5-0002-Move-page-reader-out-of-XLogReadRecord.patch
The remaining part of the main work. Eliminates callback calls
from XLogReadRecord. Applies to current master. Passes all
regression and TAP tests.
v5-0003-Change-policy-of-XLog-read-buffer-allocation.patch
Separate patch to move page buffer allocation from
XLogReaderAllocation from allers of XLogReadRecord.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 067c3bbb9105c391b7c19cc9602a5df6db5fb434 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v5 1/3] Move callback-call from ReadPageInternal to
XLogReadRecord.
WAL reader facility used to call given page-reader callback function
in the ReadPageInternal, which is two levels below from
ReadRecord. That makes things a bit complex, but furthermore we are
going to have additional callbacks for encryption, compression or
something like that works before or after reading pages. Just adding
them as new callback makes things messier. If the caller of the
current ReadRecord could call page fetching function directly, things
would get quite easier.
As the first step of that change, this patch moves the place where
that callbacks are called by 1 level above
ReadPageInternal. XLogPageRead uses a loop over new function
XLogNeedData and the callback directly instead of ReadPageInternal.
---
src/backend/access/transam/xlog.c | 15 +-
src/backend/access/transam/xlogreader.c | 355 ++++++++++++++++---------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 4 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 30 ++-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 292 insertions(+), 161 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e651a841bb..acd4ed89c4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -11521,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11580,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11675,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11689,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index c6faf48d24..1d8d470158 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool includes_page_header);
static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3);
static void ResetDecoder(XLogReaderState *state);
@@ -100,7 +100,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -224,7 +224,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -276,15 +275,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -310,8 +315,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -388,14 +393,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(total_len - gotlen + SizeOfXLogShortPHD,
+ XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -424,20 +436,38 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
+ if (state->readLen < pageHeaderSize)
+ {
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize,
+ false))
+ {
+ if (!state->read_page(state,
+ state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+ }
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
+ if (state->readLen < pageHeaderSize + len)
+ {
+ if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len,
+ true))
+ {
+ if (!state->read_page(state,
+ state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+ }
memcpy(buffer, (char *) contdata, len);
buffer += len;
@@ -468,9 +498,16 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -504,7 +541,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -513,120 +550,181 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. includes_page_header indicates that
+ * the requested region contains page header.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or error is
+ * found. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall fill the buffer at
+ * least with that portion of data and set state->readLen to the actual length
+ * of loaded data before the next call to this function.
+ *
+ * If reqLen does not contain page header, includes_page_header should be
+ * true. This function internally adds page header to reqLen.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool includes_page_header)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
- targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
- reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (includes_page_header)
+ {
+ /*
+ * Include page header length in request, but the total shoudn't
+ * exceed the block size.
+ */
+ uint32 headerLen =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ if (reqLen + headerLen <= XLOG_BLCKSZ)
+ addLen = headerLen;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->readSegNo,
+ state->wal_segment_size);
+
+ /*
+ * calculate additional length for page header so that the total length
+ * doesn't exceed the block size.
+ */
+ if (includes_page_header)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
+ targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the readSegNo, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->readSegNo = targetSegNo;
- state->readOff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -904,7 +1002,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -912,7 +1009,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -921,8 +1018,15 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
targetPagePtr = tmpRecPtr - targetRecOff;
/* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff, true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
@@ -930,8 +1034,15 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
pageHeaderSize = XLogPageHeaderSize(header);
/* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize, false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* skip over potential continuation data */
@@ -989,7 +1100,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1fc39333f1..dad9074b9f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readOff;
+ state->wal_segment_size + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..7210a940bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23870a25a5..cc35e2a04d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 63c3879ead..4df53964e4 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -235,7 +235,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -290,7 +290,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -303,7 +304,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -316,13 +318,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b95d467805..96d1f36ebc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index aa9bc63725..030f56802b 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -34,7 +34,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -123,6 +123,18 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
/* ----------------------------------------
* Decoded representation of current record
@@ -149,17 +161,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
+ bool page_verified;
XLogSegNo readSegNo;
- uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
@@ -216,8 +220,8 @@ extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
-/* Invalidate read state */
-extern void XLogReaderInvalReadState(XLogReaderState *state);
+/* Discard bufferd page */
+extern void XLogReaderDiscardReadingPage(XLogReaderState *state);
#ifdef FRONTEND
extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..0842af9f95 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..8e52b1f4aa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
From 6228af08158ac05950fb11e2c27e703ec87b6d58 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:20:18 +0900
Subject: [PATCH v5 2/3] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL reading. Moves
page-reader calls out of XLogReadRecord. Since it is essential to
reading additional data while reading a record, the function
necessarily ask caller to do so while keeping working state. Thus the
function now works as a state machine.
---
src/backend/access/transam/twophase.c | 13 +-
src/backend/access/transam/xlog.c | 53 +-
src/backend/access/transam/xlogreader.c | 678 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 14 +-
src/backend/replication/logical/logical.c | 17 +-
src/backend/replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 83 ++-
src/bin/pg_waldump/pg_waldump.c | 26 +-
src/include/access/xlogreader.h | 90 ++--
src/include/access/xlogutils.h | 5 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 597 insertions(+), 432 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 477709bbc2..929da4eef2 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1382,19 +1382,24 @@ ParsePrepareRecord(uint8 info, char *xlrec, xl_xact_parsed_prepare *parsed)
static void
XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
{
- XLogRecord *record;
+ XLogRecord *record = NULL;
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page,
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index acd4ed89c4..96e2115aad 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,9 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *readTLI);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1195,7 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size);
if (!debug_reader)
{
@@ -4246,13 +4238,8 @@ static XLogRecord *
ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
- XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- /* Pass through parameters to XLogPageRead */
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ XLogRecord *record = NULL;
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4260,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6211,7 +6205,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6352,8 +6345,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11522,12 +11514,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ TimeLineID *readTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11570,8 +11564,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11676,6 +11670,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 1d8d470158..66bd9eb8d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -33,7 +33,7 @@
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
@@ -67,8 +67,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
- void *private_data)
+XLogReaderAllocate(int wal_segment_size)
{
XLogReaderState *state;
@@ -96,11 +95,6 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
}
state->wal_segment_size = wal_segment_size;
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
- /* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -200,209 +194,339 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
-{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. The goes to XLREAD_NEED_TOT_LEN.
+ *
+ * XLREAD_NEED_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_NEED_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_NEED_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_NEED_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
+{
+ XLogRecord *prec;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- if (RecPtr == InvalidXLogRecPtr)
- {
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
-
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
/*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
+ * Reset to the initial state anytime the caller requested new record.
*/
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
- }
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
- if (!state->page_verified)
- goto err;
-
- /*
- * We have loaded at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
+again:
+ switch (state->readRecordState)
{
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ if (RecPtr != InvalidXLogRecPtr)
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ state->record_verified = false;
+ state->readRecordState = XLREAD_NEED_TOT_LEN;
+ /* fall through */
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
+ case XLREAD_NEED_TOT_LEN:
{
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting data length doesn't contain page header. Even in
+ * the case we tell the fucntion to check counting it. We have at
+ * least the page header and required part of the first record when
+ * this function returns true.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff == 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf + state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_NEED_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_NEED_FIRST_FRAGMENT;
}
- gotheader = false;
- }
+ /* fall through */
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
+ case XLREAD_NEED_FIRST_FRAGMENT:
{
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr was corrected in the previous state */
+ if (XLogNeedData(state, targetPagePtr, request_len, false))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- do
- {
/* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_NEED_CONTINUATION;
+ }
+ /* fall through */
+
+ case XLREAD_NEED_CONTINUATION:
+ {
+ XLogRecPtr targetPagePtr;
+ char *contdata;
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ uint32 request_len;
+ uint32 record_len;
/* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ),
- false))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
- }
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ true))
+ return XLREAD_NEED_DATA;
if (!state->page_verified)
goto err;
@@ -414,8 +538,11 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
{
report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
@@ -424,121 +551,115 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* we expect there to be left.
*/
if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ pageHeader->xlp_rem_len != state->recordRemainLen)
{
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
goto err;
}
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (state->readLen < pageHeaderSize)
- {
- while (XLogNeedData(state, targetPagePtr, pageHeaderSize,
- false))
- {
- if (!state->read_page(state,
- state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
- }
- }
-
+ /*
+ * XLogNeedData should have ensured that the whole page header was
+ * read
+ */
Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
- if (state->readLen < pageHeaderSize + len)
+ request_len = record_len + pageHeaderSize;
+
+ if (state->readLen < request_len)
{
- if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len,
- true))
- {
- if (!state->read_page(state,
- state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
- }
+ /*
+ * Read the rest of the page containing the record, if we didn't
+ * get it already. (If we didn't get it yet, we'll read the page
+ * header again on next invocation. In practice, this should
+ * happen very rarely, assuming that the caller makes the whole
+ * page available to us even when we request just a part of
+ * it. request_len already contains page header.
+ */
+ if (XLogNeedData(state, targetPagePtr, request_len, false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
}
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
/* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ if (!state->record_verified)
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
goto err;
- gotheader = true;
+
+ state->record_verified = true;
}
- } while (gotlen < total_len);
- Assert(gotheader);
+ if (state->recordRemainLen > 0)
+ {
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ goto again;
+ }
+ else
+ {
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
}
-
- if (!state->page_verified)
- goto err;
-
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
-
- state->ReadRecPtr = RecPtr;
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->wal_segment_size - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderDiscardReadingPage(state);
@@ -546,7 +667,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -732,11 +854,12 @@ XLogReaderDiscardReadingPage(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -754,7 +877,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -982,12 +1105,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1018,11 +1144,9 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
targetPagePtr = tmpRecPtr - targetRecOff;
/* Read the page containing the record */
- while(XLogNeedData(state, targetPagePtr, targetRecOff, true))
+ while(XLogNeedData(state, targetPagePtr, targetRecOff, false))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
+ if (!read_page(state, private))
break;
}
@@ -1036,9 +1160,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* make sure we have enough data for the page header */
while (XLogNeedData(state, targetPagePtr, pageHeaderSize, false))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
+ if (!read_page(state, private))
break;
}
@@ -1082,11 +1204,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index dad9074b9f..d50a0ca187 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -802,8 +802,7 @@ XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +817,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -908,10 +907,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
+ TimeLineID *pageTLI = &state->readPageTLI;
XLogRecPtr read_upto,
loc;
int count;
@@ -1027,6 +1028,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index f8b9020081..11e52e4c01 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -124,7 +124,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -173,11 +173,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -232,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -374,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -482,7 +483,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 7210a940bd..5270f646bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,11 +115,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -289,7 +287,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 808a6f5b83..fb5c0a702d 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index cc35e2a04d..a4518f5b55 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,9 +762,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -2827,7 +2830,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4df53964e4..ff26b30f82 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,16 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *datadir;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -64,24 +56,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -116,17 +110,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -161,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -177,10 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -189,7 +180,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -236,11 +232,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI)
+SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char*datadir, int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *pageTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,17 +270,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
- snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
+ snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", datadir, xlogfname);
xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
@@ -324,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- *pageTLI = targetHistory[private->tliIndex].tli;
+ *pageTLI = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 96d1f36ebc..56e2f8b0b0 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -445,6 +447,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1102,13 +1105,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1131,8 +1134,17 @@ main(int argc, char **argv)
for (;;)
{
+ record = NULL;
+
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 030f56802b..c2b55ce743 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -33,14 +33,6 @@
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf,
- TimeLineID *pageTLI);
-
typedef struct
{
/* Is this block ref in use? */
@@ -70,6 +62,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_NEED_TOT_LEN,
+ XLREAD_NEED_FIRST_FRAGMENT,
+ XLREAD_NEED_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -82,46 +97,19 @@ struct XLogReaderState
*/
int wal_segment_size;
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set *pageTLI to the TLI of the file the page was
- * read from. It is currently used only for error reporting purposes, to
- * reconstruct the name of the WAL file where an error occurred.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -163,6 +151,7 @@ struct XLogReaderState
/* last read segment and segment offset for data currently in readBuf */
bool page_verified;
+ bool record_verified;
XLogSegNo readSegNo;
/*
@@ -172,8 +161,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -200,21 +187,30 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -224,7 +220,11 @@ extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
extern void XLogReaderDiscardReadingPage(XLogReaderState *state);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0842af9f95..55a9b6237a 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 31c796b765..482d3d311c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -30,6 +30,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -40,6 +44,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -96,14 +101,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 8e52b1f4aa..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page, TimeLineID *pageTLI);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From c3956235807a343e0ef3bf3e4ed4c2ae8ef882e4 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v5 3/3] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 18 ------------------
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 929da4eef2..9ec88b35ef 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 96e2115aad..425f7a12bd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6351,6 +6351,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7722,6 +7723,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 66bd9eb8d7..26f6834b8f 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -79,27 +79,11 @@ XLogReaderAllocate(int wal_segment_size)
state->max_block_id = -1;
- /*
- * Permanently allocate readBuf. We do it this way, rather than just
- * making a static array, for two reasons: (1) no need to waste the
- * storage in most instantiations of the backend; (2) a static char array
- * isn't guaranteed to have any particular alignment, whereas
- * palloc_extended() will provide MAXALIGN'd storage.
- */
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
- MCXT_ALLOC_NO_OOM);
- if (!state->readBuf)
- {
- pfree(state);
- return NULL;
- }
-
state->wal_segment_size = wal_segment_size;
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -112,7 +96,6 @@ XLogReaderAllocate(int wal_segment_size)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -136,7 +119,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 56e2f8b0b0..bdf3c81b03 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1190,6 +1191,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.16.3
Attached is new version:
- Rebased. Cleaned up
- Rebased to the current master
- Fixed a known bug in the first step patch. It caused
timeline-following failure on a standby of a promoted primary.
- Fixed confused naming and setting of the parameter
includes_paeg_header.
- Removed useless XLogNeedData call in
XLREAD_NEED_CONTINUATION. The first call to the function
ensures that all required data is loaded. Finally, every case
block has just one XLogNeedData call.
- Removed the label "again" in XLogReadRecord. It is now needed
only to repeat XLREAD_NEED_CONTINUATION state. It is naturally
writtable as a while loop.
- Ensure record == NULL when XLogReadRecord returns other than
XLREAD_SUCCESS. Previously the value was not changed in that
case and it was not intuitive behavior for callers.
- Renamed XLREAD_NEED_* to XLREAD_*.
- Removed global variables readOff, readLen, readSegNo. (0003)
Other similar variables like readFile/readSource are left alone
as they are not common states of page reader and not in
XLogReaderState.
The attched are the current status, it is separated to two
significant parts plus one for readability.
v6-0001-Move-callback-call-from-ReadPageInternal-to-XLogR.patch:
ReadPageInternal part of the patch. Moves callback calls from
ReadPageInternal up to XLogReadRecord. Rerorded commit message
and fixed the bug in v5.
v6-0002-Move-page-reader-out-of-XLogReadRecord.patch
The remaining part of the main work. Eliminates callback calls
from XLogReadRecord. Reworded commit message and fixed several
bugs.
v6-0003-Remove-globals-readSegNo-readOff-readLen.patch
Seprate patch to remove some globals that are duplicate with
members of XLogReaderState.
v6-0004-Change-policy-of-XLog-read-buffer-allocation.patch
Separate patch to move page buffer allocation from
XLogReaderAllocation from allers of XLogReadRecord.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 81d3e58a1f017f34bb654cc4f66a4b9646469349 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v6 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 336 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 4 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 26 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 265 insertions(+), 166 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6876537b62..d7f899e738 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11521,7 +11520,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11580,7 +11579,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11675,7 +11675,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11689,8 +11690,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a66e3324b1..12a52159a9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,9 +34,9 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
-static void XLogReaderInvalReadState(XLogReaderState *state);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
+static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
@@ -101,7 +101,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -225,7 +225,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -277,15 +276,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -311,8 +316,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -385,18 +390,26 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -425,21 +438,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -469,9 +475,16 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -505,7 +518,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -514,120 +527,183 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
- targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
- reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) ||
+ state->readPagePtr != pageptr || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->readSegNo,
+ state->wal_segment_size);
+
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
+ targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the readSegNo, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->readSegNo = targetSegNo;
- state->readOff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
static void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -905,7 +981,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -913,7 +988,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -921,19 +996,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
@@ -990,7 +1070,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1fc39333f1..dad9074b9f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readOff;
+ state->wal_segment_size + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..7210a940bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23870a25a5..cc35e2a04d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 63c3879ead..4df53964e4 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -235,7 +235,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -290,7 +290,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -303,7 +304,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -316,13 +318,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b95d467805..96d1f36ebc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 735b1bd2fd..6de7c19a2a 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -34,7 +34,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -123,6 +123,18 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
/* ----------------------------------------
* Decoded representation of current record
@@ -149,17 +161,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
+ bool page_verified;
XLogSegNo readSegNo;
- uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..0842af9f95 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..8e52b1f4aa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
From 1998e14fbad74aa2deb4c4bead40dd019b5a6706 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v6 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 708 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 14 +-
src/backend/replication/logical/logical.c | 17 +-
src/backend/replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 83 ++-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 ++--
src/include/access/xlogutils.h | 5 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 617 insertions(+), 435 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 477709bbc2..a736504d62 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1386,15 +1386,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page,
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index d7f899e738..34ab4ea359 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,9 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *readTLI);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1195,7 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6210,7 +6205,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6351,8 +6345,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11521,12 +11514,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ TimeLineID *readTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11569,8 +11564,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11675,6 +11670,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 12a52159a9..e00fa270ae 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,7 +38,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -68,8 +68,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
- void *private_data)
+XLogReaderAllocate(int wal_segment_size)
{
XLogReaderState *state;
@@ -97,11 +96,6 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
}
state->wal_segment_size = wal_segment_size;
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
- /* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -201,321 +195,464 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
-{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
+{
+ XLogRecord *prec;
+
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- if (RecPtr == InvalidXLogRecPtr)
- {
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
-
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
/*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
+ * Reset to the initial state anytime the caller requested new record.
*/
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
+
+ switch (state->readRecordState)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf,
- &state->readPageTLI))
- break;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (!state->page_verified)
- goto err;
-
- /*
- * We have loaded at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (RecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
-
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert (pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
-
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
-
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
- goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
- break;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- if (!state->page_verified)
- goto err;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- state->ReadRecPtr = RecPtr;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->wal_segment_size - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderDiscardReadingPage(state);
@@ -523,7 +660,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -711,11 +849,12 @@ XLogReaderDiscardReadingPage(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -733,7 +872,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -961,12 +1100,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -999,9 +1141,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf,
- &state->readPageTLI))
+ if (!read_page(state, private))
break;
}
@@ -1052,11 +1192,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index dad9074b9f..d50a0ca187 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -802,8 +802,7 @@ XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +817,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -908,10 +907,12 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
+ TimeLineID *pageTLI = &state->readPageTLI;
XLogRecPtr read_upto,
loc;
int count;
@@ -1027,6 +1028,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index f8b9020081..11e52e4c01 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -124,7 +124,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -173,11 +173,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -232,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -374,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -482,7 +483,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 7210a940bd..5270f646bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,11 +115,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -289,7 +287,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 808a6f5b83..fb5c0a702d 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index cc35e2a04d..a4518f5b55 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,9 +762,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -2827,7 +2830,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4df53964e4..ff26b30f82 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,16 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *datadir;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -64,24 +56,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -116,17 +110,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -161,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -177,10 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.datadir = datadir;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -189,7 +180,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -236,11 +232,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
- TimeLineID *pageTLI)
+SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char*datadir, int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
+ TimeLineID *pageTLI = &xlogreader->readPageTLI;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,17 +270,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
- snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
+ snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", datadir, xlogfname);
xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
@@ -324,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- *pageTLI = targetHistory[private->tliIndex].tli;
+ *pageTLI = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 96d1f36ebc..a61a5a91cb 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -422,10 +422,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -445,6 +447,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1102,13 +1105,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1132,7 +1135,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6de7c19a2a..11ae82b96d 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -33,14 +33,6 @@
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf,
- TimeLineID *pageTLI);
-
typedef struct
{
/* Is this block ref in use? */
@@ -70,6 +62,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -82,46 +97,19 @@ struct XLogReaderState
*/
int wal_segment_size;
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set *pageTLI to the TLI of the file the page was
- * read from. It is currently used only for error reporting purposes, to
- * reconstruct the name of the WAL file where an error occurred.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -163,6 +151,7 @@ struct XLogReaderState
/* last read segment and segment offset for data currently in readBuf */
bool page_verified;
+ bool record_verified;
XLogSegNo readSegNo;
/*
@@ -172,8 +161,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -200,28 +187,41 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0842af9f95..55a9b6237a 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,10 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page,
- TimeLineID *pageTLI);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 31c796b765..482d3d311c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -30,6 +30,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -40,6 +44,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -96,14 +101,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 8e52b1f4aa..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page, TimeLineID *pageTLI);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From 540c683f9702a0e27a593d29019e125304a6ea7a Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v6 3/4] Remove globals readSegNo, readOff, readLen
These global variables is functionally duplicated with them in
XLogReaderState. Remove the globals.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++++++---------------------
src/include/access/xlogreader.h | 2 +-
2 files changed, 36 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 34ab4ea359..c20d3b1418 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7494,7 +7492,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11514,14 +11513,15 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
- TimeLineID *readTLI = &xlogreader->readPageTLI;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
+ TimeLineID *readTLI = &state->readPageTLI;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11534,7 +11534,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11542,10 +11542,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11555,7 +11555,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11564,17 +11564,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11602,37 +11599,33 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
*readTLI = curFileTLI;
@@ -11663,15 +11656,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11680,14 +11673,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11719,7 +11711,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 11ae82b96d..aca15be216 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -122,6 +122,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
/* ----------------------------------------
@@ -152,7 +153,6 @@ struct XLogReaderState
/* last read segment and segment offset for data currently in readBuf */
bool page_verified;
bool record_verified;
- XLogSegNo readSegNo;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
--
2.16.3
From a6f261d8454e61184142af91b0bf2a4811d78182 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v6 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 18 ------------------
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index a736504d62..358951c232 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c20d3b1418..a199600399 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index e00fa270ae..1f98dd0168 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -80,27 +80,11 @@ XLogReaderAllocate(int wal_segment_size)
state->max_block_id = -1;
- /*
- * Permanently allocate readBuf. We do it this way, rather than just
- * making a static array, for two reasons: (1) no need to waste the
- * storage in most instantiations of the backend; (2) a static char array
- * isn't guaranteed to have any particular alignment, whereas
- * palloc_extended() will provide MAXALIGN'd storage.
- */
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
- MCXT_ALLOC_NO_OOM);
- if (!state->readBuf)
- {
- pfree(state);
- return NULL;
- }
-
state->wal_segment_size = wal_segment_size;
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -113,7 +97,6 @@ XLogReaderAllocate(int wal_segment_size)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -137,7 +120,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index a61a5a91cb..5eba802720 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1188,6 +1189,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.16.3
Hello.
709d003fbd hit this. Rebased.
Works fine but needs detailed verification and maybe further
cosmetic fixes.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 6be22b10c9df068c53c98ad3106e1bd88e07aeeb Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v7 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 332 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 259 insertions(+), 162 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6c69eb6dd7..5dcb2e500c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11522,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11581,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11676,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11690,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 27c27303d6..c2bb664f07 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,9 +34,9 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
-static void XLogReaderInvalReadState(XLogReaderState *state);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
+static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
@@ -104,7 +104,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,15 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -331,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -405,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -445,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -489,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -525,7 +535,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -534,120 +544,183 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) ||
+ state->readPagePtr != pageptr || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
static void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -925,7 +998,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -933,7 +1005,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -941,19 +1013,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
@@ -1010,7 +1086,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index eb4a98cc91..0809ceaeb8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.16.3
From f570eef98799c4a6ad392a9e8771807051158987 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v7 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 702 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
src/backend/replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 74 +--
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 91 ++--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 615 insertions(+), 420 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 546bd43ce8..ced11bbdae 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1385,15 +1385,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5dcb2e500c..b06e381d73 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,8 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1194,8 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6210,7 +6205,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6351,9 +6345,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11522,12 +11514,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11570,8 +11563,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11676,6 +11669,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index c2bb664f07..9932ba3882 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,7 +38,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -68,8 +68,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -100,9 +99,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
/* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
@@ -221,318 +217,464 @@ WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
-{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
+{
+ XLogRecord *prec;
+
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- if (RecPtr == InvalidXLogRecPtr)
- {
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
-
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
/*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
+ * Reset to the initial state anytime the caller requested new record.
*/
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
+
+ switch (state->readRecordState)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (!state->page_verified)
- goto err;
-
- /*
- * We have loaded at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (RecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
-
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert (pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
-
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
-
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
- goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- if (!state->page_verified)
- goto err;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- state->ReadRecPtr = RecPtr;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderDiscardReadingPage(state);
@@ -540,7 +682,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -728,11 +871,12 @@ XLogReaderDiscardReadingPage(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -750,7 +894,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,12 +1122,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1016,8 +1163,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1068,11 +1214,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index a19726a96e..a2bc7b9fba 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -802,8 +802,7 @@ XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +817,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -908,9 +907,11 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
int count;
@@ -1025,6 +1026,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index da265f5294..db3e8f9dc0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -124,7 +124,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -173,11 +173,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -232,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -374,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -482,7 +483,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 310cd9d8cf..5270f646bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,11 +115,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -289,7 +287,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 42da631823..74e213b8e1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0809ceaeb8..21e6c47317 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,9 +761,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -2827,7 +2830,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 8aecd1adc7..c5e5d75a87 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,14 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,23 +56,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -113,16 +110,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -157,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -173,9 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -184,7 +180,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -231,10 +232,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -267,14 +269,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -319,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6e424bd8e1..ee49712830 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -407,10 +407,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -430,6 +432,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1088,13 +1091,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1118,7 +1121,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8b747d465f..8e39b5c1f6 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,13 +49,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -85,6 +78,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -92,45 +108,19 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -144,7 +134,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -183,8 +175,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -211,15 +201,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -229,15 +226,21 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
int segsize, const char *waldir);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 47b65463f9..55a9b6237a 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 31c796b765..482d3d311c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -30,6 +30,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -40,6 +44,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -96,14 +101,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 54291221c3..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From 8a933e3991ab98bc4497a9ecd77c9d262fd66cb0 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v7 3/4] Remove globals readSegNo, readOff, readLen
These global variables is functionally duplicated with them in
XLogReaderState. Remove the globals.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++++++---------------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b06e381d73..49e8ca486e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7494,7 +7492,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11514,13 +11513,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11533,7 +11533,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11541,10 +11541,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11554,7 +11554,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11563,17 +11563,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11601,40 +11598,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11662,15 +11655,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11679,14 +11672,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11718,7 +11710,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8e39b5c1f6..dc84d39f60 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -133,6 +133,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.16.3
From c5dbdb0f633f5e4b03e3c18d285c91fc106ace9c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v7 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9932ba3882..b8ad4ffcd7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.16.3
At Wed, 25 Sep 2019 15:50:32 +0900 (Tokyo Standard Time), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
<20190925.155032.13779064.horikyota.ntt@gmail.com>
> 709d003fbd hit this. Rebased.
Oops! I found a silly silent bug that it doesn't verify the first
page in new segments. Moreover it didin't load the first page in
a new loaded segment.
- Fixed a bug that it didn't load the first segment once new
loaded segment is loaded.
- Fixed a bug that it didn't verify the first segment if it is
not the target page.
Some fishy codes are reaminig but I'll post once the fixed
version.
regares.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From d511d7b9db24450596c82973a380fbae3b2d6ea1 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v8 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 306 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
src/test/recovery/t/011_crash_recovery.pl | 1 +
11 files changed, 239 insertions(+), 157 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6c69eb6dd7..5dcb2e500c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11522,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11581,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11676,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11690,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 27c27303d6..900a628752 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,8 +34,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -104,7 +104,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,15 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -331,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -405,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -445,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -489,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -534,109 +544,158 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when
+ * we are verifying the first page of new segment. In that case, skip
+ * further verification and immediately load the target page.
+ */
+ if (pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -645,9 +704,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -925,7 +982,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -933,7 +989,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -941,19 +997,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index eb4a98cc91..0809ceaeb8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..c78912571e 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -55,6 +55,7 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
# Crash and restart the postmaster
$node->stop('immediate');
+print "HOGEEEEEEEEEEEE\n";
$node->start;
# Make sure we really got a new xid
--
2.16.3
From 70d6c8543ff3d23cbb916f67ec49b2c5d8c43f75 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v8 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 702 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
src/backend/replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 74 +--
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 91 ++--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 615 insertions(+), 420 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 546bd43ce8..ced11bbdae 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1385,15 +1385,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5dcb2e500c..b06e381d73 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,8 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1194,8 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6210,7 +6205,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6351,9 +6345,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11522,12 +11514,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11570,8 +11563,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11676,6 +11669,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 900a628752..9cab82e76d 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,7 +38,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -68,8 +68,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -100,9 +99,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
/* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
@@ -221,318 +217,464 @@ WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
-{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
+{
+ XLogRecord *prec;
+
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- if (RecPtr == InvalidXLogRecPtr)
- {
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
-
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
/*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
+ * Reset to the initial state anytime the caller requested new record.
*/
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
+
+ switch (state->readRecordState)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (RecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
-
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert (pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
-
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
-
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
- goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- if (!state->page_verified)
- goto err;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- state->ReadRecPtr = RecPtr;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -540,7 +682,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -712,11 +855,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -734,7 +878,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -962,12 +1106,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1000,8 +1147,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1052,11 +1198,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index a19726a96e..a2bc7b9fba 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -802,8 +802,7 @@ XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +817,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -908,9 +907,11 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
int count;
@@ -1025,6 +1026,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index da265f5294..db3e8f9dc0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -124,7 +124,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -173,11 +173,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -232,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -374,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -482,7 +483,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 310cd9d8cf..5270f646bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,11 +115,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -289,7 +287,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 42da631823..74e213b8e1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0809ceaeb8..21e6c47317 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,9 +761,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -2827,7 +2830,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 8aecd1adc7..c5e5d75a87 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,14 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,23 +56,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -113,16 +110,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -157,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -173,9 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -184,7 +180,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -231,10 +232,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -267,14 +269,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -319,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6e424bd8e1..ee49712830 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -407,10 +407,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -430,6 +432,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1088,13 +1091,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1118,7 +1121,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8b747d465f..8e39b5c1f6 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,13 +49,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -85,6 +78,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -92,45 +108,19 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -144,7 +134,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -183,8 +175,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -211,15 +201,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -229,15 +226,21 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
int segsize, const char *waldir);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 47b65463f9..55a9b6237a 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 31c796b765..482d3d311c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -30,6 +30,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -40,6 +44,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -96,14 +101,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 54291221c3..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.16.3
From 5bdf57efcfd7d005a44830b3de7ef36b7534990e Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v8 3/4] Remove globals readSegNo, readOff, readLen
These global variables is functionally duplicated with them in
XLogReaderState. Remove the globals.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++++++---------------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b06e381d73..49e8ca486e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7494,7 +7492,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11514,13 +11513,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11533,7 +11533,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11541,10 +11541,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11554,7 +11554,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11563,17 +11563,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11601,40 +11598,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11662,15 +11655,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11679,14 +11672,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11718,7 +11710,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8e39b5c1f6..dc84d39f60 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -133,6 +133,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.16.3
From a16a51838371b8c3d2c4b444212588927b48e87d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v8 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9cab82e76d..80434aed2e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.16.3
Rebased.
I intentionally left duplicate code in XLogNeedData but changed my
mind to remove it. It makes the function small and clearer.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 7c4ce152d248546c8f56057febae6b17b6fa71bb Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v9 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 273 ++++++++++--------
src/backend/access/transam/xlogutils.c | 10 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
src/test/recovery/t/011_crash_recovery.pl | 1 +
11 files changed, 213 insertions(+), 150 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b602e28f61..a7630c643a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11540,7 +11539,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11599,7 +11598,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11694,7 +11694,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11708,8 +11709,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index c8b0d2303d..66f3bc5597 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,8 +34,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -244,7 +244,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -296,15 +295,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +334,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +408,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +455,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +492,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +543,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +684,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +962,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +969,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +977,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index b0ebe5039c..3ea71a0f84 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..c78912571e 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -55,6 +55,7 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
# Crash and restart the postmaster
$node->stop('immediate');
+print "HOGEEEEEEEEEEEE\n";
$node->start;
# Make sure we really got a new xid
--
2.23.0
From 8636422b2d519529e70968d08085053869143792 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v9 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 658 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 76 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 91 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 594 insertions(+), 399 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 546bd43ce8..ced11bbdae 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1385,15 +1385,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a7630c643a..ce3dcbb399 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,8 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1194,8 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6216,7 +6211,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6357,9 +6351,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11540,12 +11532,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11588,8 +11581,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11694,6 +11687,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 66f3bc5597..ce7fd55496 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,7 +38,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -68,8 +68,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -100,9 +99,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -220,318 +216,464 @@ WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
+ /*
+ * Reset to the initial state anytime the caller requested new record.
+ */
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
- if (RecPtr == InvalidXLogRecPtr)
+ switch (state->readRecordState)
{
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
+ if (RecPtr != InvalidXLogRecPtr)
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
- state->currRecPtr = RecPtr;
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- if (!state->page_verified)
- goto err;
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- do
+ case XLREAD_FIRST_FRAGMENT:
{
- int rest_len = total_len - gotlen;
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
}
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * The record continues on the next page. Need to reassemble
+ * record
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
{
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert (pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ state->readRecordState = XLREAD_CONTINUATION;
+ }
+ /* fall through */
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
- Assert(gotheader);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- if (!state->page_verified)
- goto err;
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- state->ReadRecPtr = RecPtr;
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -539,7 +681,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -692,11 +835,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -714,7 +858,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -942,12 +1086,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -980,8 +1127,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1032,11 +1178,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index a19726a96e..a2bc7b9fba 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -802,8 +802,7 @@ XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +817,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -908,9 +907,11 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
int count;
@@ -1025,6 +1026,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index da265f5294..db3e8f9dc0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -124,7 +124,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -173,11 +173,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -232,7 +233,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -374,7 +375,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -482,7 +483,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 310cd9d8cf..5270f646bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,11 +115,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -289,7 +287,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 42da631823..74e213b8e1 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 3ea71a0f84..c98e1f58f3 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,9 +761,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -2827,7 +2830,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 8aecd1adc7..c5e5d75a87 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,14 +41,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,23 +56,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -113,16 +110,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -157,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -173,9 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -184,7 +180,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -231,10 +232,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -267,14 +269,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
-
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
+
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -319,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6e424bd8e1..ee49712830 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -407,10 +407,12 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
if (private->endptr != InvalidXLogRecPtr)
@@ -430,6 +432,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1088,13 +1091,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1118,7 +1121,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8b747d465f..8e39b5c1f6 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,13 +49,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -85,6 +78,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -92,45 +108,19 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -144,7 +134,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -183,8 +175,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -211,15 +201,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -229,15 +226,21 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
int segsize, const char *waldir);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Functions for decoding an XLogRecord */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 47b65463f9..55a9b6237a 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 31c796b765..482d3d311c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -30,6 +30,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -40,6 +44,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -96,14 +101,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 54291221c3..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.23.0
From 3011628f8294ef225cba53c3c7e6925ffe949d91 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v9 3/4] Remove globals readSegNo, readOff, readLen
These global variables is functionally duplicated with them in
XLogReaderState. Remove the globals.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ce3dcbb399..e8a4c7916b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7512,7 +7510,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11532,13 +11531,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11551,7 +11551,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11559,10 +11559,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11572,7 +11572,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11581,17 +11581,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11619,40 +11616,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11680,15 +11673,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11697,14 +11690,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11736,7 +11728,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 8e39b5c1f6..dc84d39f60 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -133,6 +133,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.23.0
From 0fe31a828eb49c7b7c521292fc995bff4b31f7c0 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v9 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e8a4c7916b..0bc7549b07 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6355,6 +6355,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7739,6 +7740,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ce7fd55496..12a097c3eb 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -104,7 +104,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -117,7 +116,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -141,7 +139,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.23.0
At Thu, 24 Oct 2019 14:51:01 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> Rebased.
0dc8ead463 hit this. Rebased.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From d9007aee88f7400b0f03ced1b80584964a1b0b79 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v11 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 14 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 216 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5f0ee50092..4a6bd6e002 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11537,7 +11536,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11596,7 +11595,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11691,7 +11691,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11705,8 +11706,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 67418b05f1..388662ade4 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,14 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +544,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +685,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +963,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +970,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +978,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 446760ed6e..060fbc0c4c 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -813,7 +813,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -915,7 +915,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -933,7 +934,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index aa2106b152..4b0e8ea773 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index ac9209747a..e2d64cc2e5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1d03375a3e..795e84ff9f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -228,7 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -283,7 +283,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -296,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -309,13 +311,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 30a5851d87..9eece44626 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,9 +324,9 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+static bool
+XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
+ XLogRecPtr targetPtr, char *readBuff)
{
XLogDumpPrivate *private = state->private_data;
int count = XLOG_BLCKSZ;
@@ -341,7 +341,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -366,7 +367,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
@@ -1027,7 +1029,7 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
&private);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0193611b7f..d990432ffe 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,7 +49,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -131,6 +131,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -157,13 +171,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0572b24192..0867ef0599 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.23.0
From 668e7fdd3c666f9a10ee3b549858645407156c05 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v11 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 658 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 76 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 91 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 594 insertions(+), 399 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b3ad0d08e3..b6c7125aa1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,15 +1330,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 4a6bd6e002..a4e606cebf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,8 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1194,8 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6199,7 +6194,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6354,9 +6348,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11537,12 +11529,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11585,8 +11578,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11691,6 +11684,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 388662ade4..9acc8d03e2 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -221,318 +217,464 @@ WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
+ /*
+ * Reset to the initial state anytime the caller requested new record.
+ */
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
- if (RecPtr == InvalidXLogRecPtr)
+ switch (state->readRecordState)
{
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
+ if (RecPtr != InvalidXLogRecPtr)
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
- state->currRecPtr = RecPtr;
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- if (!state->page_verified)
- goto err;
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- do
+ case XLREAD_FIRST_FRAGMENT:
{
- int rest_len = total_len - gotlen;
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
}
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * The record continues on the next page. Need to reassemble
+ * record
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
{
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert (pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ state->readRecordState = XLREAD_CONTINUATION;
+ }
+ /* fall through */
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
- Assert(gotheader);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- if (!state->page_verified)
- goto err;
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- state->ReadRecPtr = RecPtr;
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -540,7 +682,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -693,11 +836,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -715,7 +859,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -943,12 +1087,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -981,8 +1128,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1033,11 +1179,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 060fbc0c4c..4b3d757677 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -696,7 +695,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -814,9 +813,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -934,6 +935,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7e06615864..d3a92044b5 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -228,7 +229,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -370,7 +371,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -478,7 +479,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 4b0e8ea773..bb21110a5b 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -107,11 +107,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -281,7 +279,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 46e6dd4d12..05fc9ebf29 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2d64cc2e5..291a8414c6 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -763,9 +763,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2783,7 +2786,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 795e84ff9f..4236ea1d12 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -39,14 +39,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -60,23 +54,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -111,16 +108,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -155,7 +154,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -171,9 +169,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -182,7 +178,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -229,10 +230,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -265,14 +267,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
-
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
+
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -317,7 +319,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9eece44626..c7c05edcda 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -325,10 +325,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+XLogDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -367,6 +369,7 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1029,13 +1032,13 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &XLogDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1059,7 +1062,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!XLogDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index d990432ffe..2494b2416f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -48,13 +48,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -84,6 +77,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -91,45 +107,19 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -143,7 +133,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -183,8 +175,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -211,15 +201,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -247,15 +244,21 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
int segsize, const char *waldir);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/*
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0867ef0599..46beb35bc6 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 6879a2e6d2..11c22f1501 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 54291221c3..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.23.0
From dac5958d17473c39ba9bab393d065e0f53dd9b6b Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v11 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a4e606cebf..364b1948e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7509,7 +7507,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11529,13 +11528,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11548,7 +11548,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11556,10 +11556,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11569,7 +11569,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11578,17 +11578,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11616,40 +11613,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11677,15 +11670,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11694,14 +11687,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11733,7 +11725,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 2494b2416f..f6249a5844 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -132,6 +132,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.23.0
From 98473703ab60b985fdc5246f94a68f9d6b6d3415 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v11 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b6c7125aa1..e69b6abae7 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9acc8d03e2..cb47aa239a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c7c05edcda..5a40047733 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1035,6 +1035,7 @@ main(int argc, char **argv)
xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1115,6 +1116,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.23.0
On Wed, Nov 27, 2019 at 12:09:23PM +0900, Kyotaro Horiguchi wrote: > 0dc8ead463 hit this. Rebased. Note: Moved to next CF. -- Michael
Вложения
On 2019-Nov-27, Kyotaro Horiguchi wrote: > At Thu, 24 Oct 2019 14:51:01 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > > Rebased. > > 0dc8ead463 hit this. Rebased. Please review the pg_waldump.c hunks in 0001; they revert recent changes. -- Álvaro Herrera https://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
At Wed, 27 Nov 2019 01:11:40 -0300, Alvaro Herrera <alvherre@2ndquadrant.com> wrote in > On 2019-Nov-27, Kyotaro Horiguchi wrote: > > > At Thu, 24 Oct 2019 14:51:01 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > > > Rebased. > > > > 0dc8ead463 hit this. Rebased. > > Please review the pg_waldump.c hunks in 0001; they revert recent changes. Ughhh! I'l check it. Thank you for noticing!! At Wed, 27 Nov 2019 12:57:47 +0900, Michael Paquier <michael@paquier.xyz> wrote in > Note: Moved to next CF. Thanks. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
At Thu, 28 Nov 2019 21:37:03 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> > > 0dc8ead463 hit this. Rebased.
> >
> > Please review the pg_waldump.c hunks in 0001; they revert recent changes.
>
> Ughhh! I'l check it. Thank you for noticing!!
Fixed that, re-rebased and small comment and cosmetic changes in this
version.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 27d9a619bb23cc3175658e34e2fec01eb545b40f Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v12 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 213 insertions(+), 150 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5f0ee50092..4a6bd6e002 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11537,7 +11536,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11596,7 +11595,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11691,7 +11691,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11705,8 +11706,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 67418b05f1..063223701e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,14 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +544,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +685,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +963,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +970,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +978,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 446760ed6e..060fbc0c4c 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -813,7 +813,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -915,7 +915,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -933,7 +934,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index aa2106b152..4b0e8ea773 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 8bafa65e50..554c186ae1 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1d03375a3e..795e84ff9f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -228,7 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -283,7 +283,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -296,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -309,13 +311,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 30a5851d87..df13cf3173 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,7 +324,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -341,7 +341,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -366,7 +367,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0193611b7f..d990432ffe 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,7 +49,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -131,6 +131,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -157,13 +171,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0572b24192..0867ef0599 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.23.0
From 59166862a9b3372d915fe013e0bf80256a1357a3 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v12 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 50 +-
src/backend/access/transam/xlogreader.c | 658 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 76 +-
src/bin/pg_waldump/pg_waldump.c | 25 +-
src/include/access/xlogreader.h | 91 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 595 insertions(+), 399 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 529976885f..f519a5b6a2 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,15 +1330,20 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
- record = XLogReadRecord(xlogreader, lsn, &errormsg);
+ while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 4a6bd6e002..a4e606cebf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -803,13 +803,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -884,8 +877,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1194,8 +1187,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4247,11 +4239,7 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (RecPtr != InvalidXLogRecPtr);
+ bool randAccess = (RecPtr != InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4259,8 +4247,15 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, RecPtr, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+ }
- record = XLogReadRecord(xlogreader, RecPtr, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6199,7 +6194,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6354,9 +6348,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11537,12 +11529,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11585,8 +11578,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11691,6 +11684,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 063223701e..71984f00f7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -221,318 +217,464 @@ WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/*
* Attempt to read an XLOG record.
*
- * If RecPtr is valid, try to read a record at that position. Otherwise
- * try to read a record just after the last one previously read.
+ * When starting to read a new record, valid RecPtr starts reading the record
+ * at that position. If invalid RecPtr is given try to start reading a record
+ * just after the last one previously read. Anytime (means in any internal
+ * state) when valid new RecPtr is given, starts reading the record at that
+ * position. This function may return XLREAD_NEED_DATA several times before
+ * returning a result record. The caller shall read in some new data then call
+ * this function again with the same parameters.
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecord **record,
+ char **errormsg)
{
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
+ /*
+ * Reset to the initial state anytime the caller requested new record.
+ */
+ if (RecPtr != InvalidXLogRecPtr && RecPtr != state->ReadRecPtr)
+ state->readRecordState = XLREAD_NEXT_RECORD;
- if (RecPtr == InvalidXLogRecPtr)
+ switch (state->readRecordState)
{
- /* No explicit start point; read the record after the one we just read */
- RecPtr = state->EndRecPtr;
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if (state->ReadRecPtr == InvalidXLogRecPtr)
- randAccess = true;
+ if (RecPtr != InvalidXLogRecPtr)
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, the passed-in record pointer should already be
+ * pointing to a valid record starting position.
+ */
+ state->ReadRecPtr = RecPtr;
- /*
- * RecPtr is pointing to end+1 of the previous WAL record. If we're
- * at a page boundary, no more records can fit on the current page. We
- * must skip over the page header, but we can't do that until we've
- * read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, the passed-in record pointer should already be
- * pointing to a valid record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset ReadRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+ else
+ {
+ /*
+ * Otherwise, read the record after the one we just read. (Or
+ * the first record, if this is the first call. In that case,
+ * EndRecPtr was set to the desired starting point above.)
+ *
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header on the next
+ * page, but we can't do that until we've read in the page,
+ * since the header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
- state->currRecPtr = RecPtr;
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- if (!state->page_verified)
- goto err;
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- do
+ case XLREAD_FIRST_FRAGMENT:
{
- int rest_len = total_len - gotlen;
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
}
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * The record continues on the next page. Need to reassemble
+ * record
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
{
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert(pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ state->readRecordState = XLREAD_CONTINUATION;
+ }
+ /* fall through */
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ while(state->recordRemainLen > 0)
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
- Assert(gotheader);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- if (!state->page_verified)
- goto err;
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
- state->ReadRecPtr = RecPtr;
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -540,7 +682,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -693,11 +836,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -715,7 +859,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -943,12 +1087,15 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* debugging purposes.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogReaderState saved_state = *state;
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -981,8 +1128,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1033,11 +1179,19 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- while (XLogReadRecord(state, tmpRecPtr, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, tmpRecPtr, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
/* continue after the record */
tmpRecPtr = InvalidXLogRecPtr;
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 060fbc0c4c..4b3d757677 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -696,7 +695,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -814,9 +813,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -934,6 +935,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7e06615864..d3a92044b5 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -228,7 +229,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -370,7 +371,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -478,7 +479,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, startptr, &err);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 4b0e8ea773..bb21110a5b 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -107,11 +107,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -281,7 +279,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, startptr, &errm);
+ while (XLogReadRecord(ctx->reader, startptr, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 46e6dd4d12..05fc9ebf29 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -433,7 +433,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, startlsn, &errm);
+ while (XLogReadRecord(ctx->reader, startlsn, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 554c186ae1..1d811b05d5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -763,9 +763,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2780,7 +2783,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, logical_startptr, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader,
+ logical_startptr, &record, &errm) == XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
logical_startptr = InvalidXLogRecPtr;
/* xlog record was invalid */
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 795e84ff9f..4236ea1d12 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -39,14 +39,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -60,23 +54,26 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
do
{
- record = XLogReadRecord(xlogreader, startpoint, &errormsg);
+ while (XLogReadRecord(xlogreader, startpoint, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
- XLogRecPtr errptr;
+ XLogRecPtr errptr = xlogreader->EndRecPtr;
- errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
+ if (startpoint)
+ errptr = startpoint;
if (errormsg)
pg_fatal("could not read WAL record at %X/%X: %s",
@@ -111,16 +108,18 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
- record = XLogReadRecord(xlogreader, ptr, &errormsg);
+ while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -155,7 +154,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -171,9 +169,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -182,7 +178,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
{
uint8 info;
- record = XLogReadRecord(xlogreader, searchptr, &errormsg);
+ while (XLogReadRecord(xlogreader, searchptr, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -229,10 +230,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -265,14 +267,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
-
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
+
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -317,7 +319,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index df13cf3173..16ee68efbe 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -325,10 +325,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -367,6 +369,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1029,13 +1032,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1059,7 +1063,14 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
+ while (XLogReadRecord(xlogreader_state,
+ first_record, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index d990432ffe..2494b2416f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -48,13 +48,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -84,6 +77,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -91,45 +107,19 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next, if XLogReadRecord receives an invalid recptr.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -143,7 +133,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -183,8 +175,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -211,15 +201,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -247,15 +244,21 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
int segsize, const char *waldir);
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- XLogRecPtr recptr, char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecPtr recptr,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/*
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0867ef0599..46beb35bc6 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 6879a2e6d2..11c22f1501 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 54291221c3..25fa68d5b9 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.23.0
From 7014c499b3fb64c5d1e6980d2adc3584f694910d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v12 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a4e606cebf..364b1948e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -783,14 +783,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -877,10 +873,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7509,7 +7507,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11529,13 +11528,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11548,7 +11548,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11556,10 +11556,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11569,7 +11569,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11578,17 +11578,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11616,40 +11613,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11677,15 +11670,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11694,14 +11687,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11733,7 +11725,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 2494b2416f..f6249a5844 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -132,6 +132,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.23.0
From a1153f804da2125d1a023975afe00202caa5dee5 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v12 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f519a5b6a2..e72d5c7ccb 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 71984f00f7..81a95623a2 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
do
{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 16ee68efbe..4a6491007f 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1036,6 +1036,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1116,6 +1117,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.23.0
On 29/11/2019 10:14, Kyotaro Horiguchi wrote:
> At Thu, 28 Nov 2019 21:37:03 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
>>>> 0dc8ead463 hit this. Rebased.
>>>
>>> Please review the pg_waldump.c hunks in 0001; they revert recent changes.
>>
>> Ughhh! I'l check it. Thank you for noticing!!
>
> Fixed that, re-rebased and small comment and cosmetic changes in this
> version.
Thanks! I finally got around to look at this again. A lot has happened
since I last looked at this. Notably, commit 0dc8ead463 introduced
another callback function into the XLogReader interface. It's not
entirely clear what the big picture with the new callback was and how
that interacts with the refactoring here. I'll have to spend some time
to make myself familiar with those changes.
Earlier in this thread, you wrote:
>
> - Changed calling convention of XLogReadRecord
>
> To make caller loop simple, XLogReadRecord now allows to specify
> the same valid value while reading the a record. No longer need
> to change lsn to invalid after the first call in the following
> reader loop.
>
> while (XLogReadRecord(state, lsn, &record, &errormsg) == XLREAD_NEED_DATA)
> {
> if (!page_reader(state))
> break;
> }
Actually, I had also made a similar change in the patch version I posted
at
https://www.postgresql.org/message-id/4f7a5fad-fa04-b0a3-231b-56d200b646dc%40iki.fi.
Maybe you missed that email? It looks like you didn't include any of the
changes from that in the patch series. In any case, clearly that idea
has some merit, since we both independently made a change in that
calling convention :-).
I changed that by adding new function, XLogBeginRead(), that repositions
the reader, and removed the 'lsn' argument from XLogReadRecord()
altogether. All callers except one in findLastCheckPoint() pg_rewind.c
positioned the reader once, and then just read sequentially from there,
so I think that API is more convenient. With that, the usage looks
something like this:
state = XLogReaderAllocate (...)
XLogBeginRead(state, start_lsn);
while (ctx->reader->EndRecPtr < end_of_wal)
{
while (XLogReadRecord(state, &record, &errormsg) == XLREAD_NEED_DATA)
{
if (!page_reader(state))
break;
}
/* do stuff */
....
}
Actually, I propose that we make that change first, and then continue
reviewing the rest of these patches. I think it's a more convenient
interface, independently of the callback refactoring. What do you think
of the attached patch?
- Heikki
Вложения
On 2020-Jan-17, Heikki Linnakangas wrote: > I changed that by adding new function, XLogBeginRead(), that repositions the > reader, and removed the 'lsn' argument from XLogReadRecord() altogether. All > callers except one in findLastCheckPoint() pg_rewind.c positioned the reader > once, and then just read sequentially from there, so I think that API is > more convenient. I like it. +1 -- Álvaro Herrera https://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
Thanks!
At Fri, 17 Jan 2020 20:14:36 +0200, Heikki Linnakangas <hlinnaka@iki.fi> wrote in
> On 29/11/2019 10:14, Kyotaro Horiguchi wrote:
> > At Thu, 28 Nov 2019 21:37:03 +0900 (JST), Kyotaro Horiguchi
> > <horikyota.ntt@gmail.com> wrote in
> >>>> 0dc8ead463 hit this. Rebased.
> >>>
> >>> Please review the pg_waldump.c hunks in 0001; they revert recent
> >>> changes.
> >>
> >> Ughhh! I'l check it. Thank you for noticing!!
> > Fixed that, re-rebased and small comment and cosmetic changes in this
> > version.
>
> Thanks! I finally got around to look at this again. A lot has happened
> since I last looked at this. Notably, commit 0dc8ead463 introduced
> another callback function into the XLogReader interface. It's not
> entirely clear what the big picture with the new callback was and how
> that interacts with the refactoring here. I'll have to spend some time
> to make myself familiar with those changes.
>
> Earlier in this thread, you wrote:
> > - Changed calling convention of XLogReadRecord
> > To make caller loop simple, XLogReadRecord now allows to specify
> > the same valid value while reading the a record. No longer need
> > to change lsn to invalid after the first call in the following
> > reader loop.
> > while (XLogReadRecord(state, lsn, &record, &errormsg) ==
> > XLREAD_NEED_DATA)
> > {
> > if (!page_reader(state))
> > break;
> > }
>
> Actually, I had also made a similar change in the patch version I
> posted at
> https://www.postgresql.org/message-id/4f7a5fad-fa04-b0a3-231b-56d200b646dc%40iki.fi. Maybe
> you missed that email? It looks like you didn't include any of the
> changes from that in the patch series. In any case, clearly that idea
> has some merit, since we both independently made a change in that
> calling convention :-).
I'm very sorry but I missed that...
> Actually, I propose that we make that change first, and then continue
> reviewing the rest of these patches. I think it's a more convenient
> interface, independently of the callback refactoring. What do you
> think of the attached patch?
Separating XLogBeginRead seems reasonable. The annoying recptr trick
is no longer needed. In particular some logical-decoding stuff become
far cleaner by the patch.
fetching_ckpt of ReadRecord is a bit annoying but that coundn't be
moved outside to, say, ReadCheckpointRecord in a clean way. (Anyway
XLogBeginRead is not the place, of course).
As I looked through it, it looks good to me but give me some more time
to look closer.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
Hello. At Mon, 20 Jan 2020 17:24:07 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > Separating XLogBeginRead seems reasonable. The annoying recptr trick > is no longer needed. In particular some logical-decoding stuff become > far cleaner by the patch. > > fetching_ckpt of ReadRecord is a bit annoying but that coundn't be > moved outside to, say, ReadCheckpointRecord in a clean way. (Anyway > XLogBeginRead is not the place, of course). > > As I looked through it, it looks good to me but give me some more time > to look closer. It seems to me that it works perfectly, and everything looks good to me except one point. - * In this case, the passed-in record pointer should already be + * In this case, EndRecPtr record pointer should already be I'm not confident but isn't the "record pointer" redundant? EndRecPtr seems containing that meaning, and the other occurances of that kind of variable names are not accompanied by that. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
On Tue, 21 Jan 2020 at 18:46, Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote: > > Hello. > > At Mon, 20 Jan 2020 17:24:07 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > > Separating XLogBeginRead seems reasonable. The annoying recptr trick > > is no longer needed. In particular some logical-decoding stuff become > > far cleaner by the patch. > > > > fetching_ckpt of ReadRecord is a bit annoying but that coundn't be > > moved outside to, say, ReadCheckpointRecord in a clean way. (Anyway > > XLogBeginRead is not the place, of course). > > > > As I looked through it, it looks good to me but give me some more time > > to look closer. > > It seems to me that it works perfectly, and everything looks good I seem to remember some considerable pain in this area when it came to timeline switches. Especially with logical decoding and xlog records that split across a segment boundary. My first attempts at logical decoding timeline following appeared fine and passed tests until they were put under extended real world workloads, at which point they exploded when they tripped corner cases like this. I landed up writing ridiculous regression tests to trigger some of these behaviours. I don't recall how many of them made it into the final patch to core but it's worth a look in the TAP test suite. -- Craig Ringer http://www.2ndQuadrant.com/ 2ndQuadrant - PostgreSQL Solutions for the Enterprise
On 21/01/2020 12:45, Kyotaro Horiguchi wrote: > At Mon, 20 Jan 2020 17:24:07 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in >> Separating XLogBeginRead seems reasonable. The annoying recptr trick >> is no longer needed. In particular some logical-decoding stuff become >> far cleaner by the patch. >> >> fetching_ckpt of ReadRecord is a bit annoying but that coundn't be >> moved outside to, say, ReadCheckpointRecord in a clean way. (Anyway >> XLogBeginRead is not the place, of course). >> >> As I looked through it, it looks good to me but give me some more time >> to look closer. > > It seems to me that it works perfectly, and everything looks good to > me except one point. > > - * In this case, the passed-in record pointer should already be > + * In this case, EndRecPtr record pointer should already be > > I'm not confident but isn't the "record pointer" redundant? EndRecPtr > seems containing that meaning, and the other occurances of that kind > of variable names are not accompanied by that. I fixed that, fixed the XLogFindNextRecord() function so that it positions the reader like XLogBeginRead() does (I had already adjusted the comments to say that, but the function didn't actually do it), and pushed. Thanks for the review! I'll continue looking at the callback API changes on Monday. - Heikki
On 21/01/2020 13:33, Craig Ringer wrote: > On Tue, 21 Jan 2020 at 18:46, Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote: >> It seems to me that it works perfectly, and everything looks good > > I seem to remember some considerable pain in this area when it came to > timeline switches. Especially with logical decoding and xlog records > that split across a segment boundary. > > My first attempts at logical decoding timeline following appeared fine > and passed tests until they were put under extended real world > workloads, at which point they exploded when they tripped corner cases > like this. > > I landed up writing ridiculous regression tests to trigger some of > these behaviours. I don't recall how many of them made it into the > final patch to core but it's worth a look in the TAP test suite. Yeah, the timeline switching stuff is complicated. The small XLogBeginRead() patch isn't really affected, but it's definitely something to watch out for in the callback API patch. If you happen to have any extra ridiculous tests still lying around, would be nice to look at them. - Heikki
At Tue, 21 Jan 2020 19:45:10 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> Hello.
>
> At Mon, 20 Jan 2020 17:24:07 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> > Separating XLogBeginRead seems reasonable. The annoying recptr trick
> > is no longer needed. In particular some logical-decoding stuff become
> > far cleaner by the patch.
> >
> > fetching_ckpt of ReadRecord is a bit annoying but that coundn't be
> > moved outside to, say, ReadCheckpointRecord in a clean way. (Anyway
> > XLogBeginRead is not the place, of course).
> >
> > As I looked through it, it looks good to me but give me some more time
> > to look closer.
>
> It seems to me that it works perfectly, and everything looks good to
> me except one point.
>
> - * In this case, the passed-in record pointer should already be
> + * In this case, EndRecPtr record pointer should already be
>
> I'm not confident but isn't the "record pointer" redundant? EndRecPtr
> seems containing that meaning, and the other occurances of that kind
> of variable names are not accompanied by that.
I rebased this on the 38a957316d and its follow-on patch 30012a04a6.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 2c605922be988292a80a479b4e59cf799299db19 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v13 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 282 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 218 insertions(+), 155 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6e09ded597..357432ffa2 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -883,7 +883,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4262,7 +4262,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11586,7 +11585,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11645,7 +11644,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11740,7 +11740,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11754,8 +11755,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 32f02256ed..2c1500443e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -269,7 +269,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -319,14 +318,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -352,8 +357,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -426,18 +431,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -466,21 +478,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -510,9 +515,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -555,109 +566,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -666,9 +707,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -949,7 +988,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -957,7 +995,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -965,19 +1003,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b217ffa52f..47676bf800 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +818,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -920,7 +920,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -938,7 +939,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 25b89e5616..acc8ef73be 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0c65f1660b..9f8e5a592d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803..78ee9f3faa 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -227,7 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -282,7 +282,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -295,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -308,13 +310,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 83202b5b87..fcfe80730c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -322,7 +322,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -339,7 +339,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -364,7 +365,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 67c3fa12d9..ab7941a155 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.18.2
From 337ff936d1776a18b8004e4a2fdd560d13364909 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v13 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 693 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 14 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 14 +-
src/bin/pg_rewind/parsexlog.c | 69 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
src/include/replication/logicalfuncs.h | 5 +-
14 files changed, 605 insertions(+), 419 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5adf956f41..f5f6278880 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 357432ffa2..9d5d71ac8c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -802,13 +802,6 @@ static XLogSource readSource = 0; /* XLOG_FROM_* code */
static XLogSource currentSource = 0; /* XLOG_FROM_* code */
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -883,8 +876,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1193,8 +1186,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4256,15 +4248,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4272,8 +4259,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6213,7 +6208,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6368,9 +6362,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11586,12 +11578,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11634,8 +11627,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11740,6 +11733,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 2c1500443e..ad85f50c77 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -239,6 +235,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -247,314 +244,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
- */
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
-{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
+ */
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
+{
+ XLogRecord *prec;
+
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
+ switch (state->readRecordState)
{
- /* read the record after the one we just read */
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
-
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
+ {
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert(pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
-
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
-
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
- goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- if (!state->page_verified)
- goto err;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- state->ReadRecPtr = RecPtr;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -562,7 +697,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -715,11 +851,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -737,7 +874,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -969,11 +1106,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1006,8 +1146,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1059,8 +1198,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 47676bf800..b2c2abc826 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -701,7 +700,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -819,9 +818,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -939,6 +940,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index cf93200618..d0a002fae7 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -228,7 +229,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -370,7 +371,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -477,7 +478,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index acc8ef73be..8158842432 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -107,11 +107,9 @@ check_permissions(void)
}
bool
-logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_local_xlog_page(LogicalDecodingContext *ctx)
{
- return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page);
+ return read_local_xlog_page(ctx->reader);
}
/*
@@ -279,7 +277,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 7c89694611..ccf96134a7 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -428,7 +428,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 9f8e5a592d..b9b3f07cad 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -763,9 +763,12 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(LogicalDecodingContext *ctx)
{
+ XLogReaderState *state = ctx->reader;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2794,7 +2797,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 78ee9f3faa..55337b65f1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -39,14 +39,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -60,18 +54,20 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -108,17 +104,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -153,7 +151,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -169,9 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +176,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -228,10 +228,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -264,14 +265,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -316,7 +317,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index fcfe80730c..e82f184de2 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,10 +323,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -365,6 +367,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1025,13 +1028,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1055,7 +1059,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..f0ccfc6895 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(LogicalDecodingContext *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index ab7941a155..6a477d0930 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,9 +11,6 @@
#include "replication/logical.h"
-extern bool logical_read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr,
- char *cur_page);
+extern bool logical_read_local_xlog_page(LogicalDecodingContext *ctx);
#endif
--
2.18.2
From dba58a12d19ca5cb050386c3e34aede21b354a41 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v13 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9d5d71ac8c..ce62559f42 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -782,14 +782,10 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
@@ -876,10 +872,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7526,7 +7524,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11578,13 +11577,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11597,7 +11597,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11605,10 +11605,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11618,7 +11618,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11627,17 +11627,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11665,40 +11662,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11726,15 +11719,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11743,14 +11736,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = 0;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11782,7 +11774,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e59e42bee3..a862db6d90 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -135,6 +135,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.2
From 07dd2d6767e2aae474ce720ba57818ae28c5c49e Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v13 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ce62559f42..e319793404 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6366,6 +6366,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7753,6 +7754,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d0a002fae7..9f67674a88 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -516,6 +517,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index e82f184de2..a40f1d71e4 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1032,6 +1032,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1108,6 +1109,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.2
5d0c2d5eba shot out this.
Rebased.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 6837855f0c938b5e34039897158bc912c6619d2b Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v7 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 211 insertions(+), 148 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7621fc05e2..51c409d00e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -900,7 +900,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4285,7 +4285,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11660,7 +11659,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11719,7 +11718,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11814,7 +11814,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11828,8 +11829,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 32f02256ed..2c1500443e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -269,7 +269,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -319,14 +318,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -352,8 +357,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -426,18 +431,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -466,21 +478,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -510,9 +515,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -555,109 +566,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -666,9 +707,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -949,7 +988,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -957,7 +995,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -965,19 +1003,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b217ffa52f..47676bf800 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +818,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -920,7 +920,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -938,7 +939,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 76ec3c7dd0..c66ea308d8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803..78ee9f3faa 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -227,7 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -282,7 +282,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -295,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -308,13 +310,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 279acfa044..443fe33599 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -322,7 +322,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -339,7 +339,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -364,7 +365,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
From 915f8e4af7c2aff284166eedd9c5d53bee6c4853 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v7 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 647 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 8 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 13 +-
src/bin/pg_rewind/parsexlog.c | 71 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
13 files changed, 579 insertions(+), 389 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5adf956f41..f5f6278880 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 51c409d00e..c138504f0d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -819,13 +819,6 @@ static XLogSource readSource = XLOG_FROM_ANY;
static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -900,8 +893,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1212,8 +1205,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4279,15 +4271,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4295,8 +4282,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6257,7 +6252,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6413,9 +6407,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11660,12 +11652,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11708,8 +11701,8 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11814,6 +11807,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 2c1500443e..ad85f50c77 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -239,6 +235,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -247,314 +244,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
+ switch (state->readRecordState)
+ {
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- RecPtr = state->EndRecPtr;
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
+ {
+ /* read the record after the one we just read */
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
- state->currRecPtr = RecPtr;
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- if (!state->page_verified)
- goto err;
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- do
+ case XLREAD_FIRST_FRAGMENT:
{
- int rest_len = total_len - gotlen;
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
}
/*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
+ * The record continues on the next page. Need to reassemble
+ * record
*/
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
{
+ /* We treat this as a "bogus data" condition */
report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
- Assert(pageHeaderSize <= state->readLen);
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ state->readRecordState = XLREAD_CONTINUATION;
+ }
+ /* fall through */
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ while(state->recordRemainLen > 0)
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
- Assert(gotheader);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
- }
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- if (!state->page_verified)
- goto err;
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
- state->ReadRecPtr = RecPtr;
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -562,7 +697,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -715,11 +851,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -737,7 +874,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -969,11 +1106,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1006,8 +1146,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1059,8 +1198,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 47676bf800..b2c2abc826 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -701,7 +700,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -819,9 +818,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -939,6 +940,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 5adf253583..7782e3ad2f 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -230,7 +231,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -372,7 +373,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -479,7 +480,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 04510094a8..a28e05b8aa 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -269,7 +269,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index ce0c9127bc..572c4fadda 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -449,7 +449,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c66ea308d8..b165900d4f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -763,9 +763,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2801,7 +2803,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 78ee9f3faa..55337b65f1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -39,14 +39,8 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -60,18 +54,20 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -108,17 +104,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -153,7 +151,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -169,9 +166,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -181,7 +176,12 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex))
+ break;
+ }
if (record == NULL)
{
@@ -228,10 +228,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -264,14 +265,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
-
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
+
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -316,7 +317,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 443fe33599..0cd0d132af 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,10 +323,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -365,6 +367,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1026,13 +1029,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1056,7 +1060,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..cbe9ed751c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
From fe15f23e40882b5b09a291a9fcb7b0b628b7304d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v7 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 79 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 37 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c138504f0d..a3eded30ad 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -796,18 +796,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
@@ -893,10 +889,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7586,7 +7584,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11652,13 +11651,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11671,7 +11671,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11679,10 +11679,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11692,7 +11692,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11701,17 +11701,14 @@ retry:
receivedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11739,40 +11736,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11800,15 +11793,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11817,14 +11810,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -11856,7 +11848,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e59e42bee3..a862db6d90 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -135,6 +135,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.2
From 3aacd6c3910f252b5c9d3323c3cf25b1d6fe2d85 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v7 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a3eded30ad..3f7f3e4d4f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6411,6 +6411,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7813,6 +7814,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0cd0d132af..ac6740f21d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1033,6 +1033,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1109,6 +1110,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.2
I found this conficts with a7e8ece41cf7a96d8a9c4c037cdfef304d950831.
Rebased on it.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From e67149578c750977a2a2872d3f4dbb3a86c82a6d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v8 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 278 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 218 insertions(+), 152 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 11e32733c4..b0ad9376d6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -908,7 +908,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4328,7 +4328,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11810,7 +11809,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11869,7 +11868,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11964,7 +11964,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11978,8 +11979,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 79ff976474..6250093dd9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -272,7 +272,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -322,14 +321,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -355,8 +360,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -429,18 +434,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -469,21 +481,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -513,9 +518,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -558,109 +569,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the read_page callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -669,9 +710,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -952,7 +991,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -960,7 +998,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -968,19 +1006,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 6cb143e161..2464a0d092 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -822,7 +822,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -924,7 +924,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -942,7 +943,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0e933228fc..e1c1de22c5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -798,7 +798,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -818,7 +818,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -848,7 +851,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 14a5db5433..1b0199f52f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -231,7 +231,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -291,7 +291,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -304,7 +305,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -320,7 +324,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -333,13 +338,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d7bd9ccac2..33fde23e7b 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,7 +323,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -340,7 +340,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -365,7 +366,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
From c85848858bc09d82128283f52c1ecfa091e87086 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v8 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 681 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 8 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 13 +-
src/bin/pg_rewind/parsexlog.c | 80 +-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
13 files changed, 601 insertions(+), 410 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 2f7d4ed59a..24b08810ac 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b0ad9376d6..e380eaa186 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -827,13 +827,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -908,8 +901,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1221,8 +1214,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4322,15 +4314,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4338,8 +4325,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6309,7 +6304,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6465,9 +6459,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11810,12 +11802,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11858,8 +11851,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11964,6 +11957,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 6250093dd9..7c8c6fc314 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -242,6 +238,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -250,314 +247,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -565,7 +700,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -718,11 +854,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -740,7 +877,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -972,11 +1109,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1009,8 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1062,8 +1201,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 2464a0d092..82f1ed2d1a 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -823,9 +822,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,6 +944,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 5adf253583..7782e3ad2f 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -230,7 +231,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -372,7 +373,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -479,7 +480,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f5384f1df8..8f2b2dd1fe 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -269,7 +269,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f776de3df7..bd34042458 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -489,7 +489,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e1c1de22c5..0dab43545d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -799,9 +799,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2834,7 +2836,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1b0199f52f..5b16681a37 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,19 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -111,17 +108,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, NULL))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -156,7 +155,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -172,10 +170,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -185,7 +180,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -232,10 +233,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -268,14 +270,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -288,7 +290,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -302,7 +304,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -344,7 +346,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 33fde23e7b..1c42050277 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,10 +324,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -366,6 +368,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1033,13 +1036,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1063,7 +1067,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..cbe9ed751c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
From 5e828dc842764dcd13d841f38772a23c46266d9c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v8 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 79 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 37 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e380eaa186..1ef6574f5f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -800,18 +800,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
@@ -901,10 +897,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7638,7 +7636,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11802,13 +11801,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11821,7 +11821,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11829,10 +11829,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11842,7 +11842,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11851,17 +11851,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11889,40 +11886,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11950,15 +11943,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11967,14 +11960,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12006,7 +11998,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e59e42bee3..a862db6d90 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -135,6 +135,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.2
From 4f24908518d502b655fe6ee1e53e1ae97b44161c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v8 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 5b16681a37..c8b6c5ee90 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -132,6 +135,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -173,6 +177,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -223,6 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.2
At Tue, 21 Apr 2020 17:04:27 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
>
Mmm. The message body seems disappearing for uncertain reason.
cd12323440 conflicts with this. Rebased.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From b3b780c2143ae70b82fb1e8256e771f11276af31 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v9 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 278 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 218 insertions(+), 152 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 11e32733c4..b0ad9376d6 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -908,7 +908,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4328,7 +4328,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11810,7 +11809,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11869,7 +11868,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11964,7 +11964,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11978,8 +11979,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 79ff976474..6250093dd9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -272,7 +272,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -322,14 +321,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -355,8 +360,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -429,18 +434,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -469,21 +481,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -513,9 +518,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -558,109 +569,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the read_page callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -669,9 +710,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -952,7 +991,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -960,7 +998,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -968,19 +1006,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 6cb143e161..2464a0d092 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -822,7 +822,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -924,7 +924,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -942,7 +943,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 0e933228fc..e1c1de22c5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -798,7 +798,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -818,7 +818,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -848,7 +851,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c51b5db315..6cbe108129 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -233,7 +233,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -293,7 +293,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -306,7 +307,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -322,7 +326,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -335,13 +340,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d7bd9ccac2..33fde23e7b 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -323,7 +323,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -340,7 +340,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -365,7 +366,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
From ff5cacf30066734993a2b89911f30da2e68eea09 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v9 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 11 +-
src/backend/access/transam/xlog.c | 54 +-
src/backend/access/transam/xlogreader.c | 681 +++++++++++-------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/logical/logical.c | 17 +-
.../replication/logical/logicalfuncs.c | 8 +-
src/backend/replication/slotfuncs.c | 8 +-
src/backend/replication/walsender.c | 13 +-
src/bin/pg_rewind/parsexlog.c | 81 ++-
src/bin/pg_waldump/pg_waldump.c | 24 +-
src/include/access/xlogreader.h | 90 +--
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 9 +-
13 files changed, 601 insertions(+), 411 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 2f7d4ed59a..24b08810ac 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,8 +1330,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1339,7 +1338,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b0ad9376d6..e380eaa186 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -827,13 +827,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -908,8 +901,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1221,8 +1214,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!debug_reader)
{
@@ -4322,15 +4314,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4338,8 +4325,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6309,7 +6304,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6465,9 +6459,7 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- &XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11810,12 +11802,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11858,8 +11851,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -11964,6 +11957,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 6250093dd9..7c8c6fc314 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -70,8 +70,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, const char *waldir)
{
XLogReaderState *state;
@@ -102,9 +101,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- state->read_page = pagereadfunc;
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -242,6 +238,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -250,314 +247,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the read_page callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -565,7 +700,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -718,11 +854,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -740,7 +877,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -972,11 +1109,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1009,8 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->read_page(state, state->readPagePtr, state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1062,8 +1201,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 2464a0d092..82f1ed2d1a 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -823,9 +822,11 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,6 +944,7 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 5adf253583..7782e3ad2f 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,7 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +169,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -230,7 +231,7 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -372,7 +373,7 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -479,7 +480,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f5384f1df8..8f2b2dd1fe 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -269,7 +269,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index f776de3df7..bd34042458 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -489,7 +489,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->read_page(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e1c1de22c5..0dab43545d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -799,9 +799,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -2834,7 +2836,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->read_page(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 6cbe108129..86b6540057 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,19 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -112,18 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -158,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -174,10 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -187,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -234,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -270,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -290,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -304,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -346,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 33fde23e7b..1c42050277 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,10 +324,12 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* XLogReader read_page callback
*/
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -366,6 +368,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1033,13 +1036,14 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
- &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1063,7 +1067,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6ad953eea3..e59e42bee3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,13 +50,6 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
-
typedef struct
{
/* Is this block ref in use? */
@@ -86,6 +79,29 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/* ----------------------------------------
@@ -93,46 +109,20 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Data input callback (mandatory).
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB read_page;
-
/*
* System identifier of the xlog files we're about to read. Set to zero
* (the default value) if unknown or unimportant.
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -146,7 +136,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -186,8 +178,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -214,15 +204,22 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
- XLogPageReadCB pagereadfunc,
- void *private_data);
+ const char *waldir);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -252,12 +249,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index dc7d894e5d..312acb4fa3 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void XLogReadDetermineTimeline(XLogReaderState *state,
XLogRecPtr wantPage, uint32 wantLength);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 3b7ca7f1da..cbe9ed751c 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogReadPageCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogReadPageCB read_page;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,14 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogPageReadCB read_page,
+ LogicalDecodingXLogReadPageCB read_page,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
From 86c61eee7fe2c51d462d3fdfb037ffd30921a014 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v9 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 79 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 37 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e380eaa186..1ef6574f5f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -800,18 +800,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
@@ -901,10 +897,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7638,7 +7636,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11802,13 +11801,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11821,7 +11821,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11829,10 +11829,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11842,7 +11842,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11851,17 +11851,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11889,40 +11886,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11950,15 +11943,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -11967,14 +11960,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12006,7 +11998,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index e59e42bee3..a862db6d90 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -135,6 +135,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.2
From 4dbf455ee2336e1dabda6a735113b456efb00df5 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v9 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->read_page = read_page;
ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 86b6540057..78073f969d 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.2
At Wed, 22 Apr 2020 10:12:46 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> cd12323440 conflicts with this. Rebased.
b060dbe000 is conflicting. I gave up isolating XLogOpenSegment from
xlogreader.c, since the two are tightly coupled than I thought.
This patch removes all the three callbacks (open/close/page_read) in
XL_ROUTINE from XLogReaderState. It only has "cleanup" callback
instead.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From fd04ff06de3ab7bcc36f0c80fff42452e49e0259 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v10 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ca09d81b08..da468598e4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -909,7 +909,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4329,7 +4329,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11854,7 +11853,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11913,7 +11912,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12008,7 +12008,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12022,8 +12023,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 5995798b58..399bad0603 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -274,7 +274,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -324,14 +323,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -357,8 +362,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -431,18 +436,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -471,21 +485,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -515,9 +522,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -560,109 +574,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -671,9 +714,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -954,7 +995,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -962,7 +1002,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -970,19 +1010,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 86847cbb54..d3a94d611f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -815,7 +815,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -835,7 +835,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -863,7 +866,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index d637f5eb77..907db33b9e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index c21b0ba972..53e4212ac4 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.2
From f057f47e030cf6cf8a94e5b82237f6c95afa8f20 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v10 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 47 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 656 insertions(+), 494 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e1904877fa..8630ad2032 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index da468598e4..840218efee 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -828,13 +828,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -909,8 +902,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1222,8 +1215,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4323,15 +4315,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4339,8 +4326,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6317,7 +6312,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6477,13 +6471,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11854,12 +11844,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11902,8 +11893,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12008,6 +11999,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 399bad0603..bdde75086c 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -71,7 +71,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -82,7 +82,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -105,8 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -138,8 +136,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -244,6 +242,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -252,317 +251,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -570,7 +704,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -722,11 +857,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -744,7 +880,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -976,11 +1112,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1013,9 +1152,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1067,8 +1204,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1088,9 +1233,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1103,6 +1248,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1134,10 +1280,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index dc69e5ce5f..d99474c173 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6fed3cfd23..cac197045d 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -465,9 +464,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -492,7 +490,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a94d611f..cc7e7fdac7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -293,11 +293,9 @@ InitWalSender(void)
if (!am_db_walsender)
{
xlogreader = &fake_xlogreader;
- xlogreader->routine =
- *XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close);
WALOpenSegmentInit(&xlogreader->seg, &xlogreader->segcxt,
wal_segment_size, NULL);
+ xlogreader->cleanup_cb = wal_segment_close;
}
}
@@ -816,9 +814,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -846,11 +846,11 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
- state->seg.ws_tli, /* Pass the current TLI because only
+ xlogreader->seg.ws_tli, /* Pass the current TLI because only
* WalSndSegmentOpen controls whether new
* TLI is needed. */
&errinfo))
@@ -863,8 +863,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
* read() succeeds in that case, but the data we tried to read might
* already have been overwritten with new WAL records.
*/
- XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
- CheckXLogRemoved(segno, state->seg.ws_tli);
+ XLByteToSeg(targetPagePtr, segno, xlogreader->segcxt.ws_segsize);
+ CheckXLogRemoved(segno, xlogreader->seg.ws_tli);
state->readLen = count;
return true;
@@ -1018,9 +1018,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1183,9 +1182,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2495,14 +2493,14 @@ WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo,
{
XLogSegNo endSegNo;
- XLByteToSeg(sendTimeLineValidUpto, endSegNo, state->segcxt.ws_segsize);
- if (state->seg.ws_segno == endSegNo)
+ XLByteToSeg(sendTimeLineValidUpto, endSegNo, xlogreader->segcxt.ws_segsize);
+ if (xlogreader->seg.ws_segno == endSegNo)
*tli_p = sendTimeLineNextTLI;
}
- XLogFilePath(path, *tli_p, nextSegNo, state->segcxt.ws_segsize);
- state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
- if (state->seg.ws_file >= 0)
+ XLogFilePath(path, *tli_p, nextSegNo, xlogreader->segcxt.ws_segsize);
+ xlogreader->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
+ if (xlogreader->seg.ws_file >= 0)
return;
/*
@@ -2763,7 +2761,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2861,7 +2859,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 907db33b9e..cad8e13965 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 53e4212ac4..1d0167e26d 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -276,12 +253,17 @@ extern void WALOpenSegmentInit(WALOpenSegment *seg, WALSegmentContext *segcxt,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -301,6 +283,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.2
From 4774cc0cff9e90abda426bdf7e0ef1050e1b81a8 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v10 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 79 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 37 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 840218efee..694307a177 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -801,18 +801,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
@@ -902,10 +898,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7662,7 +7660,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11844,13 +11843,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11863,7 +11863,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11871,10 +11871,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11884,7 +11884,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11893,17 +11893,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11931,40 +11928,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11992,15 +11985,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -12009,14 +12002,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12048,7 +12040,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1d0167e26d..79bb2bc537 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -156,6 +156,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.2
From d2f988b454a662c76bb3b681f454d7d35c639a33 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v10 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 8630ad2032..7f394156bd 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 694307a177..b144e3e241 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6477,6 +6477,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7889,6 +7890,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index bdde75086c..71a4931131 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -110,7 +110,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -123,7 +122,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -150,7 +148,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d99474c173..18d3248192 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cad8e13965..c6247344d6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.2
On Tue, 26 May 2020, 15:40 Kyotaro Horiguchi, <horikyota.ntt@gmail.com> wrote:
This patch removes all the three callbacks (open/close/page_read) in
XL_ROUTINE from XLogReaderState. It only has "cleanup" callback
instead.
I actually have a use in mind for these callbacks - to support reading WAL for logical decoding from a restore_command like tool. So we can archive wal when it's no longer required for recovery and reduce the risk of filling pg_wal if a standby lags.
I don't object to your cleanup at all. I'd like it to be properly pluggable, whereas right now it has hard coded callbacks that differ for little reason.
Just noting that the idea of a callback here isn't a bad thing.
Thank you for the comment. At Tue, 26 May 2020 20:17:47 +0800, Craig Ringer <craig@2ndquadrant.com> wrote in > On Tue, 26 May 2020, 15:40 Kyotaro Horiguchi, <horikyota.ntt@gmail.com> > wrote: > > > > > This patch removes all the three callbacks (open/close/page_read) in > > XL_ROUTINE from XLogReaderState. It only has "cleanup" callback > > instead. > > > > I actually have a use in mind for these callbacks - to support reading WAL > for logical decoding from a restore_command like tool. So we can archive > wal when it's no longer required for recovery and reduce the risk of > filling pg_wal if a standby lags. > > I don't object to your cleanup at all. I'd like it to be properly > pluggable, whereas right now it has hard coded callbacks that differ for > little reason. > > Just noting that the idea of a callback here isn't a bad thing. I agree that plugin is generally not bad as far as it were standalone, that is, as far as it is not tightly cooperative with the opposite side of the caller of it. However, actually it seems to me that the xlogreader plugins are too-deeply coupled with the callers of xlogreader in many aspects involving error-handling and retry-mechanism. As Alvaro mentioned we may have page-decrypt callback shortly as another callback of xlogreader. Xlogreader could be more messy by introducing such plugins, that actually have no business with xlogreader at all. Evidently xlogreader can be a bottom-end module (that is, a module that doesn't depend on another module). It is I think a good thing to isolate xlogreader from the changes of its callers and correlated plugins. A major problem of this patch is that the state machine used there might be another mess here, though. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
cfbot is complaining as this is no longer applicable. Rebased.
In v14, some reference to XLogReaderState parameter to read_pages
functions are accidentally replaced by the reference to the global
variable xlogreader. Fixed it, too.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From a4584ab5deddd49325c2f8e3fd78de406c95ce89 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v15 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fd93bcfaeb..e570e56a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -912,7 +912,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4332,7 +4332,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11860,7 +11859,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11919,7 +11918,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12014,7 +12014,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12028,8 +12029,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index cb76be4f46..3d599325ee 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +487,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +524,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +576,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +716,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -956,7 +997,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -964,7 +1004,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -972,19 +1012,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 322b0e8ff5..1b10b6df20 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -828,7 +828,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -930,7 +930,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -948,7 +949,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index e2477c47e0..bf2711bddd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -811,7 +811,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -831,7 +831,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -859,7 +862,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index bc6f976994..0b7e73ae79 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -236,7 +236,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -296,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -309,7 +310,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -325,7 +329,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -338,13 +343,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index d1a0678935..ab0c28d9b7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b0f2a6ed43..6c8848e14f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -201,13 +215,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
From 0667f99a70b1b5567ee42766647033e780d9a725 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v15 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 700 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/replication/logical.h | 11 +-
13 files changed, 648 insertions(+), 487 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9b2e59bf0e..b0d60a0d0f 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -831,13 +831,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -912,8 +905,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
@@ -4326,15 +4318,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4342,8 +4329,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6320,7 +6315,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool fast_promoted = false;
struct stat st;
@@ -6480,13 +6474,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11860,12 +11850,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11908,8 +11899,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12014,6 +12005,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3d599325ee..9281b57379 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,317 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u at %X/%X",
- pageHeader->xlp_rem_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -572,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -724,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -746,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -978,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1015,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1069,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1090,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1105,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1136,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1b10b6df20..303333571f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -689,8 +689,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -705,7 +704,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -792,6 +791,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -829,9 +829,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -944,11 +946,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 61902be3b0..a15b0b3355 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -120,7 +120,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -169,11 +170,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -231,7 +233,8 @@ CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -328,7 +331,7 @@ CreateInitDecodingContext(char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -376,7 +379,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -429,8 +433,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -483,7 +487,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 88033a79b2..8a678074d5 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -486,9 +485,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -513,7 +511,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index bf2711bddd..2fd24a9b14 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -812,9 +809,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -842,7 +841,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1014,9 +1013,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1179,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2761,7 +2758,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2859,7 +2856,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 0b7e73ae79..3dd6df4be6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -40,15 +40,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -62,20 +56,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -113,19 +109,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -160,7 +156,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -176,11 +171,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -190,7 +181,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -237,10 +234,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -273,14 +271,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -293,7 +291,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -307,7 +305,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -349,7 +347,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab0c28d9b7..02e71138cc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1042,16 +1048,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1075,7 +1079,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6c8848e14f..6a7fe140cb 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -227,8 +199,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -255,6 +225,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -262,9 +241,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -272,12 +249,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -297,6 +279,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index c2f2475e5d..a743db49cf 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -95,14 +100,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
From f0d25e4ebe4017f0786da477e6c8f1014aa7c9bf Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 79 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 37 insertions(+), 43 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f9b0108602..06e7ff4a24 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
@@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7665,7 +7663,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11850,13 +11849,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11899,17 +11899,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11937,40 +11934,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -11998,15 +11991,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -12015,14 +12008,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12054,7 +12046,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 6a7fe140cb..09fc692fa1 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -156,6 +156,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.4
From 00af90e3b1ab3f1e708b37a7c13c1464450ac6ce Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v15 4/4] Change policy of XLog read-buffer allocation
Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
src/backend/access/transam/twophase.c | 2 ++
src/backend/access/transam/xlog.c | 2 ++
src/backend/access/transam/xlogreader.c | 3 ---
src/backend/replication/logical/logical.c | 2 ++
src/bin/pg_rewind/parsexlog.c | 6 ++++++
src/bin/pg_waldump/pg_waldump.c | 2 ++
6 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b0d60a0d0f..5955bab699 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, lsn);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
}
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 06e7ff4a24..04478e78bc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6480,6 +6480,7 @@ StartupXLOG(void)
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory"),
errdetail("Failed while allocating a WAL reading processor.")));
+ xlogreader->readBuf = palloc(XLOG_BLCKSZ);
xlogreader->system_identifier = ControlFile->system_identifier;
/*
@@ -7892,6 +7893,7 @@ StartupXLOG(void)
close(readFile);
readFile = -1;
}
+ pfree(xlogreader->readBuf);
XLogReaderFree(xlogreader);
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9281b57379..388bddaf41 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -112,7 +112,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
{
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -125,7 +124,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
if (!allocate_recordbuf(state, 0))
{
pfree(state->errormsg_buf);
- pfree(state->readBuf);
pfree(state);
return NULL;
}
@@ -152,7 +150,6 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
- pfree(state->readBuf);
pfree(state);
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index a15b0b3355..31f1029000 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
ReorderBufferFree(ctx->reorder);
FreeSnapshotBuilder(ctx->snapshot_builder);
+ pfree(ctx->reader->readBuf);
XLogReaderFree(ctx->reader);
MemoryContextDelete(ctx->context);
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 3dd6df4be6..26cba673f6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, startpoint);
do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
} while (xlogreader->ReadRecPtr != endpoint);
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
XLogBeginRead(xlogreader, ptr);
while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
}
endptr = xlogreader->EndRecPtr;
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
+ xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
searchptr = forkptr;
for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
searchptr = record->xl_prev;
}
+ pg_free(xlogreader->readBuf);
XLogReaderFree(xlogreader);
if (xlogreadfd != -1)
{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
/* first find a valid recptr to start from */
first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
(uint32) xlogreader_state->ReadRecPtr,
errormsg);
+ pfree(xlogreader_state->readBuf);
XLogReaderFree(xlogreader_state);
return EXIT_SUCCESS;
--
2.18.4
Hi,
I applied your v15 patchset to master ed2c7f65bd9f15f8f7cd21ad61602f983b1e72e9. Here are three feedback points for you:
= 1. Build error when WAL_DEBUG is defined manually =
>>>>>>>>
gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wcast-function-type -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-format-truncation -Wno-stringop-truncation -O2 -I../../../../src/include -D_GNU_SOURCE -c -o xlog.o xlog.c
In file included from /usr/include/x86_64-linux-gnu/bits/types/stack_t.h:23,
from /usr/include/signal.h:303,
from ../../../../src/include/storage/sinval.h:17,
from ../../../../src/include/access/xact.h:22,
from ../../../../src/include/access/twophase.h:17,
from xlog.c:33:
xlog.c: In function ‘XLogInsertRecord’:
xlog.c:1219:56: error: called object is not a function or function pointer
1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
| ^~~~
xlog.c:1219:19: error: too few arguments to function ‘XLogReaderAllocate’
1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
| ^~~~~~~~~~~~~~~~~~
In file included from ../../../../src/include/access/clog.h:14,
from xlog.c:25:
../../../../src/include/access/xlogreader.h:243:25: note: declared here
243 | extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
| ^~~~~~~~~~~~~~~~~~
make[4]: *** [<builtin>: xlog.o] Error 1
<<<<<<<<
The following chunk in 0002 seems to be the cause of the error. There is no comma between two NULLs.
>>>>>>>>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
(..snipped..)
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
<<<<<<<<
= 2. readBuf allocation in XLogReaderAllocate =
I applied your v15 patchset to master ed2c7f65bd9f15f8f7cd21ad61602f983b1e72e9. Here are three feedback points for you:
= 1. Build error when WAL_DEBUG is defined manually =
How to reproduce:
$ sed -i -E -e 's|^/\* #define WAL_DEBUG \*/$|#define WAL_DEBUG|' src/include/pg_config_manual.h
$ ./configure && make
$ ./configure && make
Expected: PostgreSQL is successfully made.
Actual: I got the following make error:
>>>>>>>>
gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wcast-function-type -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-format-truncation -Wno-stringop-truncation -O2 -I../../../../src/include -D_GNU_SOURCE -c -o xlog.o xlog.c
In file included from /usr/include/x86_64-linux-gnu/bits/types/stack_t.h:23,
from /usr/include/signal.h:303,
from ../../../../src/include/storage/sinval.h:17,
from ../../../../src/include/access/xact.h:22,
from ../../../../src/include/access/twophase.h:17,
from xlog.c:33:
xlog.c: In function ‘XLogInsertRecord’:
xlog.c:1219:56: error: called object is not a function or function pointer
1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
| ^~~~
xlog.c:1219:19: error: too few arguments to function ‘XLogReaderAllocate’
1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
| ^~~~~~~~~~~~~~~~~~
In file included from ../../../../src/include/access/clog.h:14,
from xlog.c:25:
../../../../src/include/access/xlogreader.h:243:25: note: declared here
243 | extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
| ^~~~~~~~~~~~~~~~~~
make[4]: *** [<builtin>: xlog.o] Error 1
<<<<<<<<
The following chunk in 0002 seems to be the cause of the error. There is no comma between two NULLs.
>>>>>>>>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e570e56a24..f9b0108602 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
(..snipped..)
@@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
if (!debug_reader)
{
<<<<<<<<
= 2. readBuf allocation in XLogReaderAllocate =
AFAIU, not XLogReaderAllocate() itself but its caller is now responsible for allocating XLogReaderState->readBuf. However, the following code still remains in src/backend/access/transam/xlogreader.c:
>>>>>>>>
74 XLogReaderState *
75 XLogReaderAllocate(int wal_segment_size, const char *waldir,
76 WALSegmentCleanupCB cleanup_cb)
77 {
:
98 state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
99 MCXT_ALLOC_NO_OOM);
<<<<<<<<
Is this okay?
= 3. XLOG_FROM_ANY assigned to global readSource =
Regarding the following chunk in 0003:
>>>>>>>>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6b42d9015f..bcb4ef270f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
<<<<<<<<
I think it is better to keep the line "static XLogSource readSource = XLOG_FROM_ANY;". XLOG_FROM_ANY is already defined as 0 in src/backend/access/transam/xlog.c.
Regards,
Takashi
>>>>>>>>
74 XLogReaderState *
75 XLogReaderAllocate(int wal_segment_size, const char *waldir,
76 WALSegmentCleanupCB cleanup_cb)
77 {
:
98 state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
99 MCXT_ALLOC_NO_OOM);
<<<<<<<<
Is this okay?
= 3. XLOG_FROM_ANY assigned to global readSource =
Regarding the following chunk in 0003:
>>>>>>>>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6b42d9015f..bcb4ef270f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
-static XLogSource readSource = XLOG_FROM_ANY;
+static XLogSource readSource = 0; /* XLOG_FROM_* code */
/*
* Keeps track of which source we're currently reading from. This is
<<<<<<<<
I think it is better to keep the line "static XLogSource readSource = XLOG_FROM_ANY;". XLOG_FROM_ANY is already defined as 0 in src/backend/access/transam/xlog.c.
Regards,
Takashi
2020年7月2日(木) 13:53 Kyotaro Horiguchi <horikyota.ntt@gmail.com>:
cfbot is complaining as this is no longer applicable. Rebased.
In v14, some reference to XLogReaderState parameter to read_pages
functions are accidentally replaced by the reference to the global
variable xlogreader. Fixed it, too.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
Takashi Menjo <takashi.menjo@gmail.com>
Thank you for the comment, Menjo-san, and noticing me of that, Michael.
Sorry for late reply.
At Fri, 17 Jul 2020 14:14:44 +0900, Takashi Menjo <takashi.menjo@gmail.com> wrote in
> Hi,
>
> I applied your v15 patchset to master
> ed2c7f65bd9f15f8f7cd21ad61602f983b1e72e9. Here are three feedback points
> for you:
>
>
> = 1. Build error when WAL_DEBUG is defined manually =
> How to reproduce:
>
> $ sed -i -E -e 's|^/\* #define WAL_DEBUG \*/$|#define WAL_DEBUG|'
> src/include/pg_config_manual.h
> $ ./configure && make
>
> Expected: PostgreSQL is successfully made.
> Actual: I got the following make error:
>
> >>>>>>>>
> gcc -Wall -Wmissing-prototypes -Wpointer-arith
> -Wdeclaration-after-statement -Werror=vla -Wendif-labels
> -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wcast-function-type
> -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard
> -Wno-format-truncation -Wno-stringop-truncation -O2
> -I../../../../src/include -D_GNU_SOURCE -c -o xlog.o xlog.c
> In file included from /usr/include/x86_64-linux-gnu/bits/types/stack_t.h:23,
> from /usr/include/signal.h:303,
> from ../../../../src/include/storage/sinval.h:17,
> from ../../../../src/include/access/xact.h:22,
> from ../../../../src/include/access/twophase.h:17,
> from xlog.c:33:
> xlog.c: In function ‘XLogInsertRecord’:
> xlog.c:1219:56: error: called object is not a function or function pointer
> 1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
> | ^~~~
> xlog.c:1219:19: error: too few arguments to function ‘XLogReaderAllocate’
> 1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
> | ^~~~~~~~~~~~~~~~~~
> In file included from ../../../../src/include/access/clog.h:14,
> from xlog.c:25:
> ../../../../src/include/access/xlogreader.h:243:25: note: declared here
> 243 | extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
> | ^~~~~~~~~~~~~~~~~~
> make[4]: *** [<builtin>: xlog.o] Error 1
> <<<<<<<<
>
> The following chunk in 0002 seems to be the cause of the error. There is
> no comma between two NULLs.
>
> >>>>>>>>
> diff --git a/src/backend/access/transam/xlog.c
> b/src/backend/access/transam/xlog.c
> index e570e56a24..f9b0108602 100644
> --- a/src/backend/access/transam/xlog.c
> +++ b/src/backend/access/transam/xlog.c
> (..snipped..)
> @@ -1225,8 +1218,7 @@ XLogInsertRecord(XLogRecData *rdata,
> appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
>
> if (!debug_reader)
> - debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
> - XL_ROUTINE(), NULL);
> + debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
>
> if (!debug_reader)
> {
> <<<<<<<<
>
>
> = 2. readBuf allocation in XLogReaderAllocate =
> AFAIU, not XLogReaderAllocate() itself but its caller is now responsible
> for allocating XLogReaderState->readBuf. However, the following code still
> remains in src/backend/access/transam/xlogreader.c:
>
> >>>>>>>>
> 74 XLogReaderState *
> 75 XLogReaderAllocate(int wal_segment_size, const char *waldir,
> 76 WALSegmentCleanupCB cleanup_cb)
> 77 {
> :
> 98 state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
> 99 MCXT_ALLOC_NO_OOM);
> <<<<<<<<
>
> Is this okay?
>
>
> = 3. XLOG_FROM_ANY assigned to global readSource =
> Regarding the following chunk in 0003:
>
> >>>>>>>>
> diff --git a/src/backend/access/transam/xlog.c
> b/src/backend/access/transam/xlog.c
> index 6b42d9015f..bcb4ef270f 100644
> --- a/src/backend/access/transam/xlog.c
> +++ b/src/backend/access/transam/xlog.c
> @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0;
> * These variables are used similarly to the ones above, but for reading
> * the XLOG. Note, however, that readOff generally represents the offset
> * of the page just read, not the seek position of the FD itself, which
> - * will be just past that page. readLen indicates how much of the current
> - * page has been read into readBuf, and readSource indicates where we got
> - * the currently open file from.
> + * will be just past that page. readSource indicates where we got the
> + * currently open file from.
> * Note: we could use Reserve/ReleaseExternalFD to track consumption of
> * this FD too; but it doesn't currently seem worthwhile, since the XLOG is
> * not read by general-purpose sessions.
> */
> static int readFile = -1;
> -static XLogSegNo readSegNo = 0;
> -static uint32 readOff = 0;
> -static uint32 readLen = 0;
> -static XLogSource readSource = XLOG_FROM_ANY;
> +static XLogSource readSource = 0; /* XLOG_FROM_* code */
>
> /*
> * Keeps track of which source we're currently reading from. This is
> <<<<<<<<
>
> I think it is better to keep the line "static XLogSource readSource =
> XLOG_FROM_ANY;". XLOG_FROM_ANY is already defined as 0 in
> src/backend/access/transam/xlog.c.
>
>
> Regards,
> Takashi
>
>
>
> 2020年7月2日(木) 13:53 Kyotaro Horiguchi <horikyota.ntt@gmail.com>:
>
> > cfbot is complaining as this is no longer applicable. Rebased.
> >
> > In v14, some reference to XLogReaderState parameter to read_pages
> > functions are accidentally replaced by the reference to the global
> > variable xlogreader. Fixed it, too.
> >
> > regards.
> >
> > --
> > Kyotaro Horiguchi
> > NTT Open Source Software Center
> >
>
>
> --
> Takashi Menjo <takashi.menjo@gmail.com>
At Tue, 08 Sep 2020 11:56:29 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > Thank you for the comment, Menjo-san, and noticing me of that, Michael. I found why the message I was writing was gone from the draft folder.. Sorry for the garbage. -- Kyotaro Horiguchi NTT Open Source Software Center
Thank you for the comment and sorry for late reply.
At Fri, 17 Jul 2020 14:14:44 +0900, Takashi Menjo <takashi.menjo@gmail.com> wrote in
> Hi,
>
> I applied your v15 patchset to master
> ed2c7f65bd9f15f8f7cd21ad61602f983b1e72e9. Here are three feedback points
> for you:
>
> = 1. Build error when WAL_DEBUG is defined manually =
..
> Expected: PostgreSQL is successfully made.
> Actual: I got the following make error:
...
> 1219 | debug_reader = XLogReaderAllocate(wal_segment_size, NULL NULL);
Ah, I completely forgot about WAL_DEBUG paths. Fixed.
> = 2. readBuf allocation in XLogReaderAllocate =
> AFAIU, not XLogReaderAllocate() itself but its caller is now responsible
> for allocating XLogReaderState->readBuf. However, the following code still
> remains in src/backend/access/transam/xlogreader.c:
>
> >>>>>>>>
> 74 XLogReaderState *
> 75 XLogReaderAllocate(int wal_segment_size, const char *waldir,
> 76 WALSegmentCleanupCB cleanup_cb)
> 77 {
> :
> 98 state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
> 99 MCXT_ALLOC_NO_OOM);
> <<<<<<<<
>
> Is this okay?
Oops! That's silly. However, I put a rethink on this. The reason of
the moving of responsibility comes from the fact that the actual
subject that fills-in the buffer is the callers of xlogreader, who
knows its size. In that light it's quite strange that xlogreader
works based on the fixed size of XLOG_BLCKSZ. I don't think it is
useful just now, but I changed 0004 so that XLOG_BLCKSZ is eliminated
from xlogreader.c. Buffer allocation is restored to
XLogReaderAllocate.
(But, I'm not sure it's worth doing..)
> = 3. XLOG_FROM_ANY assigned to global readSource =
> Regarding the following chunk in 0003:
...
> -static XLogSource readSource = XLOG_FROM_ANY;
> +static XLogSource readSource = 0; /* XLOG_FROM_* code */
>
> I think it is better to keep the line "static XLogSource readSource =
> XLOG_FROM_ANY;". XLOG_FROM_ANY is already defined as 0 in
> src/backend/access/transam/xlog.c.
That seems to be a mistake while past rebasding. XLOG_FROM_ANY is the
right thing to use here.
The attached is the rebased, and fixed version.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From 84a2f6b1abe2e34b4cdfcb7da7380a2d6161b292 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v16 1/4] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 281 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 25 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 222 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 09c01ed4ae..a91e86b290 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -909,7 +909,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4329,7 +4329,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11866,7 +11865,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11925,7 +11924,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12020,7 +12020,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12034,8 +12035,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a63ad8cfd0..f403261626 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -276,7 +276,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +325,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +364,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -433,18 +438,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -474,21 +488,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -518,9 +525,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -563,109 +577,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -674,9 +717,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -957,7 +998,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -965,7 +1005,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -973,19 +1013,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 7e915bcadf..4c6d072720 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -688,8 +688,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -827,7 +827,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -929,7 +929,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -947,7 +948,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 3f756b470a..76f94fa61f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -810,7 +810,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -830,7 +830,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -858,7 +861,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 2229c86f9a..4be2eede92 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -237,7 +237,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -297,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -310,7 +311,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -326,7 +330,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -339,13 +344,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 31e99c2a6d..da929bf4cd 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b976882229..f8f6a6264c 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,8 +57,8 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -175,6 +175,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +217,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index e59b6cf3a9..c287edf206 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.18.4
From 63e316bc952a4e2779ba13579b256c06c7fb713f Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v16 2/4] Move page-reader out of XLogReadRecord
This is the second step of removing callbacks from WAL record reader.
Since it is essential to take in additional data while reading a
record, the function have to ask caller for new data while keeping
working state. Thus the function is turned into a state machine.
---
src/backend/access/transam/twophase.c | 15 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 701 +++++++++++-------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 +--
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 121 ++-
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 649 insertions(+), 489 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ef4f9981e3..9a56b3487e 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,11 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1336,7 +1333,13 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
+
if (record == NULL)
ereport(ERROR,
(errcode_for_file_access(),
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a91e86b290..6ef1f817ad 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -828,13 +828,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -909,8 +902,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1222,8 +1215,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4323,15 +4315,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4339,8 +4326,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6317,7 +6312,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6477,13 +6471,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -11866,12 +11856,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11914,8 +11905,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12020,6 +12011,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f403261626..d829e57baa 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,318 +253,452 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
- * The returned pointer (or *errormsg) points to an internal buffer that's
- * valid until the next call to XLogReadRecord.
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->readPageTLI and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
+ * The returned pointer (or *errormsg) points to an internal buffer that's valid
+ * until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
- {
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- (uint32) (RecPtr >> 32), (uint32) RecPtr,
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
+ switch (state->readRecordState)
{
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
- break;
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the header
+ * size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
}
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the page,
+ * the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
{
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
goto err;
}
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
- {
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
- goto err;
- }
-
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
-
+ /* XLogNeedData has verified the page header */
Assert(pageHeaderSize <= state->readLen);
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the whole
+ * record header might not fit on this page. xl_tot_len is the first
+ * field of the struct, so it must be on this page (the records are
+ * MAXALIGNed), but we cannot access any other fields until we've
+ * verified that we got the whole header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
- Assert (pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after reading it
+ * from the next page. The xl_tot_len check is necessary here to
+ * ensure that we enter the XLREAD_CONTINUATION state below;
+ * otherwise we might fail to apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
+ state->record_verified = true;
+ }
+ else
{
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ (uint32) SizeOfXLogRecord, total_len);
goto err;
- gotheader = true;
+ }
}
- } while (gotlen < total_len);
-
- Assert(gotheader);
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Wait for the rest of the record, or the part of the record that
+ * fit on the first page if crossed a page boundary, to become
+ * available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert (targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
+ }
+
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
+
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
+
+ case XLREAD_CONTINUATION:
+ {
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+
+ /* we enter this state only if we haven't read the whole record. */
+ Assert (state->recordRemainLen > 0);
+
+ while(state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
+
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
+
+ /* this request contains page header */
+ Assert (targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
+
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
- if (!state->page_verified)
- goto err;
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of the
+ * record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
- state->ReadRecPtr = RecPtr;
+ /*
+ * XLogNeedData should have ensured that the whole page header
+ * was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /* XLogNeedData should have ensured all needed data was read */
+ Assert (state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /* Calculate pointer to beginning of next page, and continue */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
+ break;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert (!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -573,7 +706,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -725,11 +859,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -747,7 +882,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
(uint32) RecPtr);
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -979,11 +1114,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1016,9 +1154,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while(XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1070,8 +1206,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1091,9 +1235,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1106,6 +1250,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1137,10 +1282,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 4c6d072720..c43812ee94 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -688,8 +688,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -704,7 +703,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -791,6 +790,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -828,9 +828,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -943,11 +945,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 0f6af952f9..97def6f781 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -138,7 +138,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -188,11 +189,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -283,7 +285,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -386,7 +389,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -434,7 +437,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -487,8 +491,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -541,7 +545,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index b99c94e848..fcc81b358e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 1725ad0736..b013da9631 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -152,9 +152,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -507,9 +506,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -531,7 +529,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 76f94fa61f..33d952f9fd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,10 +579,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -811,9 +808,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -841,7 +840,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1013,9 +1012,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1178,9 +1176,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4be2eede92..7344a7c692 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -63,20 +57,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -114,19 +110,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -161,7 +157,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -177,11 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -191,7 +182,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -238,10 +235,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -274,14 +272,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -294,7 +292,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -308,7 +306,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -350,7 +348,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index da929bf4cd..9329a6a6d6 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1037,16 +1043,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1070,7 +1074,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index f8f6a6264c..38f9e59f99 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,35 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState {
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +136,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -187,7 +157,9 @@ struct XLogReaderState
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified;/* is the current record header verified? */
+
/* ----------------------------------------
@@ -229,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -257,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState;/* state machine state */
+ int recordGotLen; /* amount of current record that has
+ * already been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -264,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -274,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -299,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index c287edf206..482e4ced69 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index 705dc69c06..fdc41fff91 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -339,7 +339,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 45abc444b7..93f9769ab4 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -100,14 +105,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.18.4
From 3d79188a277ee025afa772dc40c93a86fc646d52 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v16 3/4] Remove globals readOff, readLen and readSegNo
The first two variables are functionally duplicate with them in
XLogReaderState. Remove the globals along with readSegNo, which
behaves in the similar way.
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
src/include/access/xlogreader.h | 1 +
2 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6ef1f817ad..eccb88b611 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -801,17 +801,13 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = XLOG_FROM_ANY;
/*
@@ -902,10 +898,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7662,7 +7660,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -11856,13 +11855,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -11875,7 +11875,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -11883,10 +11883,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->readSegNo))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -11896,7 +11896,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -11905,17 +11905,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->readSegNo))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -11943,40 +11940,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->readSegNo);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -12004,15 +11997,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -12021,14 +12014,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12060,7 +12052,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 38f9e59f99..732d463a49 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -156,6 +156,7 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ XLogSegNo readSegNo; /* Segment # for data currently in readBuf */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
bool record_verified;/* is the current record header verified? */
--
2.18.4
From ab679854595e6593cbdb25883ea3448f2d972ffe Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 21:29:32 +0900
Subject: [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize.
XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
src/backend/access/transam/twophase.c | 3 +-
src/backend/access/transam/xlog.c | 9 ++--
src/backend/access/transam/xlogreader.c | 57 ++++++++++++-----------
src/backend/replication/logical/logical.c | 5 +-
src/backend/replication/walsender.c | 4 +-
src/bin/pg_rewind/parsexlog.c | 6 +--
src/bin/pg_waldump/pg_waldump.c | 2 +-
src/include/access/xlogreader.h | 3 +-
8 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9a56b3487e..b64c61f742 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eccb88b611..ec78de5dd9 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+ {
+ debug_reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+ }
if (!debug_reader)
{
@@ -6469,8 +6472,8 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index d829e57baa..582b1dd11b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
+ state->pageSize = page_size;
/*
* Permanently allocate readBuf. We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
* isn't guaranteed to have any particular alignment, whereas
* palloc_extended() will provide MAXALIGN'd storage.
*/
- state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+ state->readBuf = (char *) palloc_extended(state->pageSize,
MCXT_ALLOC_NO_OOM);
if (!state->readBuf)
{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
* readRecordBufSize is set to the new buffer size.
*
* To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
* with. (That is enough for all "normal" records, but very large commit or
* abort records might need more space.)
*/
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
{
uint32 newSize = reclength;
- newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
- newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+ newSize += state->pageSize - (newSize % state->pageSize);
+ newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
#ifndef FRONTEND
@@ -238,6 +239,8 @@ void
XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
{
Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(state->readBuf != NULL &&
+ state->readBuf == (void *) MAXALIGN(state->readBuf));
ResetDecoder(state);
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
XLogPageHeader pageHeader;
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
/*
* Check if we have enough data. For the first record in the page,
* the requesting length doesn't contain page header.
*/
if (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
targetRecOff != 0))
return XLREAD_NEED_DATA;
@@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* verified that we got the whole header.
*/
prec = (XLogRecord *) (state->readBuf +
- state->ReadRecPtr % XLOG_BLCKSZ);
+ state->ReadRecPtr % state->pageSize);
total_len = prec->xl_tot_len;
/*
@@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* ensure that we enter the XLREAD_CONTINUATION state below;
* otherwise we might fail to apply ValidXLogRecordHeader at all.
*/
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
{
if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
state->PrevRecPtr, prec))
@@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* available
*/
targetPagePtr =
- state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
- targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+ state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+ targetRecOff = state->ReadRecPtr % state->pageSize;
- request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ request_len = Min(targetRecOff + total_len, state->pageSize);
record_len = request_len - targetRecOff;
/* ReadRecPtr contains page header */
@@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* Calculate pointer to beginning of next page */
state->recordContRecPtr = state->ReadRecPtr + record_len;
- Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+ Assert(state->recordContRecPtr % state->pageSize == 0);
state->readRecordState = XLREAD_CONTINUATION;
}
@@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
/* this request contains page header */
Assert (targetPagePtr != 0);
if (XLogNeedData(state, targetPagePtr,
- Min(state->recordRemainLen, XLOG_BLCKSZ),
+ Min(state->recordRemainLen, state->pageSize),
false))
return XLREAD_NEED_DATA;
@@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
Assert(state->readLen >= pageHeaderSize);
contdata = (char *) state->readBuf + pageHeaderSize;
- record_len = XLOG_BLCKSZ - pageHeaderSize;
+ record_len = state->pageSize - pageHeaderSize;
if (pageHeader->xlp_rem_len < record_len)
record_len = pageHeader->xlp_rem_len;
@@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
}
/* Calculate pointer to beginning of next page, and continue */
- state->recordContRecPtr += XLOG_BLCKSZ;
+ state->recordContRecPtr += state->pageSize;
}
/* targetPagePtr is pointing the last-read page here */
@@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
XLogPageHeaderSize((XLogPageHeader) state->readBuf);
addLen = pageHeaderSize;
- if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ if (reqLen + pageHeaderSize <= state->pageSize)
addLen = pageHeaderSize;
else
- addLen = XLOG_BLCKSZ - reqLen;
+ addLen = state->pageSize - reqLen;
Assert(addLen >= 0);
}
@@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ Assert((pageptr % state->pageSize) == 0);
/*
* Every time we request to load new data of a page to the caller, even if
@@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->readLen = state->pageSize;
return true;
}
@@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
int32 offset;
XLogPageHeader hdr = (XLogPageHeader) phdr;
- Assert((recptr % XLOG_BLCKSZ) == 0);
+ Assert((recptr % state->pageSize) == 0);
XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
"WAL file is from different database system: incorrect segment size in page
header");
return false;
}
- else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+ else if (longhdr->xlp_xlog_blcksz != state->pageSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_BLCKSZ in page
header");
+ "WAL file is from different database system: incorrect block size in page header");
return false;
}
}
@@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = tmpRecPtr % state->pageSize;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
*
* Note that record headers are MAXALIGN'ed
*/
- if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+ tmpRecPtr = targetPagePtr + state->pageSize;
else
{
/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 97def6f781..ee322ba796 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+ ctx->reader =
+ XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 33d952f9fd..a0914ddb25 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd)
errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
/* create xlogreader for physical replication */
- xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+ xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+ NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 7344a7c692..c5881594b7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
char *errormsg;
XLogRecPtr endptr;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+ xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9329a6a6d6..025479431d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1043,7 +1043,7 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+ XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 732d463a49..e9d7d70db7 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
* ----------------------------------------
*/
/* variables to communicate with page reader */
+ int pageSize; /* size of a page */
XLogRecPtr readPagePtr; /* page pointer to read */
int32 readLen; /* bytes requested to reader, or actual bytes
* read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
- const char *waldir,
+ int page_size, const char *waldir,
WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
--
2.18.4
On Thu, Mar 4, 2021 at 3:29 PM Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote: > A recent commot about LSN_FORMAT_ARGS conflicted this. > Just rebased. FYI I've been looking at this, and I think it's a very nice improvement. I'll post some review comments and a rebase shortly.
At Wed, 31 Mar 2021 10:00:02 +1300, Thomas Munro <thomas.munro@gmail.com> wrote in > On Thu, Mar 4, 2021 at 3:29 PM Kyotaro Horiguchi > <horikyota.ntt@gmail.com> wrote: > > A recent commot about LSN_FORMAT_ARGS conflicted this. > > Just rebased. > > FYI I've been looking at this, and I think it's a very nice > improvement. I'll post some review comments and a rebase shortly. Thanks for looking at this! regards. -- Kyotaro Horiguchi NTT Open Source Software Center
On Wed, Mar 31, 2021 at 7:17 PM Kyotaro Horiguchi
<horikyota.ntt@gmail.com> wrote:
> At Wed, 31 Mar 2021 10:00:02 +1300, Thomas Munro <thomas.munro@gmail.com> wrote in
> > On Thu, Mar 4, 2021 at 3:29 PM Kyotaro Horiguchi
> > <horikyota.ntt@gmail.com> wrote:
> > > A recent commot about LSN_FORMAT_ARGS conflicted this.
> > > Just rebased.
> >
> > FYI I've been looking at this, and I think it's a very nice
> > improvement. I'll post some review comments and a rebase shortly.
>
> Thanks for looking at this!
I rebased and pgindent-ed the first three patches and did some
testing. I think it looks pretty good, though I still need to check
the code coverage when running the recovery tests. There are three
compiler warnings from GCC when not using --enable-cassert, including
uninitialized variables: pageHeader and targetPagePtr. It looks like
they could be silenced as follows, or maybe you see a better way?
- XLogPageHeader pageHeader;
+ XLogPageHeader pageHeader = NULL;
uint32 pageHeaderSize;
- XLogRecPtr targetPagePtr;
+ XLogRecPtr targetPagePtr =
InvalidXLogRecPtr;
To summarise the patches:
0001 + 0002 get rid of the callback interface and replace it with a
state machine, making it the client's problem to supply data when it
returns XLREAD_NEED_DATA. I found this interface nicer to work with,
for my WAL decoding buffer patch (CF 2410), and I understand that the
encryption patch set can also benefit from it. Certainly when I
rebased my project on this patch set, I prefered the result.
0003 is nice global variable cleanup.
I haven't looked at 0004.
Вложения
On Wed, Apr 7, 2021 at 5:09 AM Thomas Munro <thomas.munro@gmail.com> wrote: > 0001 + 0002 get rid of the callback interface and replace it with a > state machine, making it the client's problem to supply data when it > returns XLREAD_NEED_DATA. I found this interface nicer to work with, > for my WAL decoding buffer patch (CF 2410), and I understand that the > encryption patch set can also benefit from it. Certainly when I > rebased my project on this patch set, I prefered the result. + state->readLen = pageHeaderSize; This variable is used for the XLogPageReader to say how much data it wants, but also for the caller to indicate how much data is loaded. Wouldn't it be better to split this into two variables: bytesWanted and bytesAvailable? (I admit that I spent a whole afternoon debugging after confusing myself about that, when rebasing my WAL readahead patch recently). I wonder if it would be better to have the client code access these values through functions (even if they just access the variables in a static inline function), to create a bit more separation? Something like XLogReaderGetWanted(&page_lsn, &bytes_wanted), and then XLogReaderSetAvailable(state, 42)? Just an idea.
Hi,
On 2021-04-07 05:09:53 +1200, Thomas Munro wrote:
> From 560cdfa444a3b05a0e6b8054f3cfeadf56e059fc Mon Sep 17 00:00:00 2001
> From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
> Date: Thu, 5 Sep 2019 20:21:55 +0900
> Subject: [PATCH v18 1/3] Move callback-call from ReadPageInternal to
> XLogReadRecord.
>
> The current WAL record reader reads page data using a call back
> function. Redesign the interface so that it asks the caller for more
> data when required. This model works better for proposed projects that
> encryption, prefetching and other new features that would require
> extending the callback interface for each case.
>
> As the first step of that change, this patch moves the page reader
> function out of ReadPageInternal(), then the remaining tasks of the
> function are taken over by the new function XLogNeedData().
> -static int
> +static bool
> XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
> XLogRecPtr targetRecPtr, char *readBuf)
> {
> @@ -12170,7 +12169,8 @@ retry:
> readLen = 0;
> readSource = XLOG_FROM_ANY;
>
> - return -1;
> + xlogreader->readLen = -1;
> + return false;
> }
> }
It seems a bit weird to assign to XlogReaderState->readLen inside the
callbacks. I first thought it was just a transient state, but it's
not. I think it'd be good to wrap the xlogreader->readLen assignment an
an inline function. That we can add more asserts etc over time.
> -/* pg_waldump's XLogReaderRoutine->page_read callback */
> +/*
> + * pg_waldump's WAL page rader, also used as page_read callback for
> + * XLogFindNextRecord
> + */
> static bool
> -WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
> - XLogRecPtr targetPtr, char *readBuff)
> +WALDumpReadPage(XLogReaderState *state, void *priv)
> {
> - XLogDumpPrivate *private = state->private_data;
> + XLogRecPtr targetPagePtr = state->readPagePtr;
> + int reqLen = state->readLen;
> + char *readBuff = state->readBuf;
> + XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
It seems weird to pass a void *priv to a function that now doesn't at
all need the type punning anymore.
Greetings,
Andres Freund
On 2021-Apr-07, Thomas Munro wrote: > I wonder if it would be better to have the client code access these > values through functions (even if they just access the variables in a > static inline function), to create a bit more separation? Something > like XLogReaderGetWanted(&page_lsn, &bytes_wanted), and then > XLogReaderSetAvailable(state, 42)? Just an idea. I think more opacity is good in this area, generally speaking. There are way too many globals, and they interact in nontrivial ways across the codebase. Just look at the ThisTimeLineID recent disaster. I don't have this patch sufficiently paged-in to say that bytes_wanted/ bytes_available is precisely the thing we need, but if it makes for a cleaner interface, I'm for it. This module keeps some state inside itself, and others part of the state is in its users; that's not good, and any cleanup on that is welcome. BTRW it's funny that after these patches, "xlogreader" no longer reads anything. It's more an "xlog interpreter" -- the piece of code that splits individual WAL records from a stream of WAL bytes that's caller's responsibility to obtain somehow. But (and, again, I haven't read this patch recently) it still offers pieces that support a reader, in addition to its main interface as the interpreter. Maybe it's not a totally stupid idea to split it in even more different files. -- Álvaro Herrera 39°49'30"S 73°17'W
On Wed, Apr 7, 2021 at 11:18 AM Alvaro Herrera <alvherre@2ndquadrant.com> wrote: > BTRW it's funny that after these patches, "xlogreader" no longer reads > anything. It's more an "xlog interpreter" -- the piece of code that > splits individual WAL records from a stream of WAL bytes that's caller's > responsibility to obtain somehow. But (and, again, I haven't read this > patch recently) it still offers pieces that support a reader, in > addition to its main interface as the interpreter. Maybe it's not a > totally stupid idea to split it in even more different files. Yeah, I like "decoder", and it's already called that in some places...
On 2021-Apr-07, Thomas Munro wrote: > On Wed, Apr 7, 2021 at 11:18 AM Alvaro Herrera <alvherre@2ndquadrant.com> wrote: > > BTRW it's funny that after these patches, "xlogreader" no longer reads > > anything. It's more an "xlog interpreter" -- the piece of code that > > splits individual WAL records from a stream of WAL bytes that's caller's > > responsibility to obtain somehow. But (and, again, I haven't read this > > patch recently) it still offers pieces that support a reader, in > > addition to its main interface as the interpreter. Maybe it's not a > > totally stupid idea to split it in even more different files. > > Yeah, I like "decoder", and it's already called that in some places... Yeah, that works ... -- Álvaro Herrera 39°49'30"S 73°17'W "Los trabajadores menos efectivos son sistematicamente llevados al lugar donde pueden hacer el menor daño posible: gerencia." (El principio Dilbert)
At Tue, 6 Apr 2021 16:09:55 -0700, Andres Freund <andres@anarazel.de> wrote in
> Hi,
>
> > XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
> > XLogRecPtr targetRecPtr, char *readBuf)
> > {
> > @@ -12170,7 +12169,8 @@ retry:
> > readLen = 0;
> > readSource = XLOG_FROM_ANY;
> >
> > - return -1;
> > + xlogreader->readLen = -1;
> > + return false;
> > }
> > }
>
> It seems a bit weird to assign to XlogReaderState->readLen inside the
> callbacks. I first thought it was just a transient state, but it's
> not. I think it'd be good to wrap the xlogreader->readLen assignment an
> an inline function. That we can add more asserts etc over time.
Sounds reasonable. The variable is split up into request/result
variables and setting the result variable is wrapped by a
function. (0005).
> > -/* pg_waldump's XLogReaderRoutine->page_read callback */
> > +/*
> > + * pg_waldump's WAL page rader, also used as page_read callback for
> > + * XLogFindNextRecord
> > + */
> > static bool
> > -WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
> > - XLogRecPtr targetPtr, char *readBuff)
> > +WALDumpReadPage(XLogReaderState *state, void *priv)
> > {
> > - XLogDumpPrivate *private = state->private_data;
> > + XLogRecPtr targetPagePtr = state->readPagePtr;
> > + int reqLen = state->readLen;
> > + char *readBuff = state->readBuf;
> > + XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
>
> It seems weird to pass a void *priv to a function that now doesn't at
> all need the type punning anymore.
Mmm. I omitted it since client code was somewhat out-of-scope. In the
attached 0004 WALDumpReadPage() is no longer used as the callback of
XLogFindNextRecord.
On the way fixing them, I found that XLogReaderState.readPageTLI has
been moved to XLogReaderState.seg.ws_tli so I removed it from 0001.
I haven't changed the name "XLog reader" to "XLog decoder". I'm doing
that but it affects somewhat wide range of code.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
From bda7ea561de0a8aec75eb50e8f07daaf96509583 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Thu, 5 Sep 2019 20:21:55 +0900
Subject: [PATCH v19 1/5] Move callback-call from ReadPageInternal to
XLogReadRecord.
The current WAL record reader reads page data using a call back
function. Redesign the interface so that it asks the caller for more
data when required. This model works better for proposed projects that
encryption, prefetching and other new features that would require
extending the callback interface for each case.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal(), then the remaining tasks of the
function are taken over by the new function XLogNeedData().
Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp>
Author: Heikki Linnakangas <hlinnaka@iki.fi>
Reviewed-by: Antonin Houska <ah@cybertec.at>
Reviewed-by: Alvaro Herrera <alvherre@2ndquadrant.com>
Reviewed-by: Takashi Menjo <takashi.menjo@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 325 +++++++++++++++---------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 21 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 32 ++-
src/include/access/xlogutils.h | 2 +-
8 files changed, 267 insertions(+), 159 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c1d4415a43..8085ca1117 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -920,7 +920,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4375,7 +4375,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -12107,7 +12106,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -12166,7 +12165,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -12261,7 +12261,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -12275,8 +12276,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 42738eb940..f2345ab09e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -261,8 +261,48 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* If the reading fails for some other reason, NULL is also returned, and
* *errormsg is set to a string with details of the failure.
*
+ * Returns XLREAD_NEED_DATA if more data is needed to finish reading the
+ * current record. In that case, state->readPagePtr and state->readLen inform
+ * the desired position and minimum length of data needed. The caller shall
+ * read in the requested data and set state->readBuf to point to a buffer
+ * containing it. The caller must also set state->seg->ws_tli and
+ * state->readLen to indicate the timeline that it was read from, and the
+ * length of data that is now available (which must be >= given readLen),
+ * respectively.
+ *
+ * If invalid data is encountered, returns XLREAD_FAIL with *record being set to
+ * NULL. *errormsg is set to a string with details of the failure.
* The returned pointer (or *errormsg) points to an internal buffer that's
* valid until the next call to XLogReadRecord.
+ *
+ *
+ * This function runs a state machine consists of the following states.
+ *
+ * XLREAD_NEXT_RECORD :
+ * The initial state, if called with valid RecPtr, try to read a record at
+ * that position. If invalid RecPtr is given try to read a record just after
+ * the last one previously read.
+ * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN.
+ *
+ * XLREAD_TOT_LEN:
+ * Examining record header. Ends after reading record total
+ * length. recordRemainLen and recordGotLen are initialized.
+ *
+ * XLREAD_FIRST_FRAGMENT:
+ * Reading the first fragment. Ends with finishing reading a single
+ * record. Goes to XLREAD_NEXT_RECORD if that's all or
+ * XLREAD_CONTINUATION if we have continuation.
+
+ * XLREAD_CONTINUATION:
+ * Reading continuation of record. Ends with finishing the whole record then
+ * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates
+ * how much is left and readRecordBuf holds the partially assert
+ * record.recordContRecPtr points to the beginning of the next page where to
+ * continue.
+ *
+ * If wrong data found in any state, the state machine stays at the current
+ * state. This behavior allows to continue reading a reacord switching among
+ * different souces, while streaming replication.
*/
XLogRecord *
XLogReadRecord(XLogReaderState *state, char **errormsg)
@@ -276,7 +316,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -326,14 +365,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -359,8 +404,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -432,18 +477,27 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr,
+ state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -473,21 +527,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert(pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -517,9 +564,16 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -562,109 +616,138 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the page_read() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the page_read callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+ }
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
+ */
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the
+ * next invocation as the caller might now be rereading data from a
+ * different source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the segment, even if we looked at parts of
- * it before, we need to do verification as the page_read callback might
- * now be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
+ * info that is present in the first page's "long" header. Don't do this
+ * if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -673,9 +756,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -953,7 +1034,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -961,27 +1041,32 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
- * least short page-header worth of data
+ * XLogNeedData() is prepared to handle that and will read at least
+ * short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->routine.page_read(state, state->readPagePtr,
+ state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert(state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index d17d660f46..46eda33f25 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -824,7 +824,7 @@ wal_segment_close(XLogReaderState *state)
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -926,7 +926,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -944,7 +945,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4bf8a18e01..a4d6f30957 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -806,7 +806,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -826,7 +826,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -854,7 +857,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 59ebac7d6a..cf119848b0 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (private->restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
/*
@@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
private->restoreCommand);
if (xlogreadfd < 0)
- return -1;
+ {
+ xlogreader->readLen = -1;
+ return false;
+ }
else
pg_log_debug("using file \"%s\" restored from archive",
xlogfpath);
@@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f8b8afe4a7..75ece5c658 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -331,7 +331,7 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/* pg_waldump's XLogReaderRoutine->page_read callback */
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -348,7 +348,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -373,7 +374,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 21d200d3df..5d9e0d3292 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,12 +57,12 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definitions for various xlogreader interactions */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
+ XLogRecPtr targetPagePtr,
+ int reqLen,
+ XLogRecPtr targetRecPtr,
+ char *readBuf);
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
@@ -175,6 +175,19 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -203,13 +216,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 9ac602b674..364a21c4ea 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
--
2.27.0
From f7c3ca4c82c753e493a41abfcc1a184203c40720 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 12:58:27 +0900
Subject: [PATCH v19 2/5] Move page-reader out of XLogReadRecord().
This is the second step of removing callbacks from the WAL decoder.
XLogReadRecord() return XLREAD_NEED_DATA to indicate that the caller
should supply new data, and the decoder works as a state machine.
Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp>
Author: Heikki Linnakangas <hlinnaka@iki.fi>
Reviewed-by: Antonin Houska <ah@cybertec.at>
Reviewed-by: Alvaro Herrera <alvherre@2ndquadrant.com>
Reviewed-by: Takashi Menjo <takashi.menjo@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/twophase.c | 14 +-
src/backend/access/transam/xlog.c | 58 +-
src/backend/access/transam/xlogreader.c | 690 ++++++++++--------
src/backend/access/transam/xlogutils.c | 17 +-
src/backend/replication/logical/logical.c | 26 +-
.../replication/logical/logicalfuncs.c | 13 +-
src/backend/replication/slotfuncs.c | 18 +-
src/backend/replication/walsender.c | 32 +-
src/bin/pg_rewind/parsexlog.c | 84 ++-
src/bin/pg_waldump/pg_waldump.c | 36 +-
src/include/access/xlogreader.h | 122 ++--
src/include/access/xlogutils.h | 4 +-
src/include/pg_config_manual.h | 2 +-
src/include/replication/logical.h | 11 +-
14 files changed, 628 insertions(+), 499 deletions(-)
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 89335b64a2..3137cb3ecc 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
char *errormsg;
TimeLineID save_currtli = ThisTimeLineID;
- xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &read_local_xlog_page,
- .segment_open = &wal_segment_open,
- .segment_close = &wal_segment_close),
- NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -1342,7 +1339,12 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
errdetail("Failed while allocating a WAL reading processor.")));
XLogBeginRead(xlogreader, lsn);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!read_local_xlog_page(xlogreader))
+ break;
+ }
/*
* Restore immediately the timeline where it was previously, as
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8085ca1117..b7d7e6d31b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -838,13 +838,6 @@ static XLogSource currentSource = XLOG_FROM_ANY;
static bool lastSourceFailed = false;
static bool pendingWalRcvRestart = false;
-typedef struct XLogPageReadPrivate
-{
- int emode;
- bool fetching_ckpt; /* are we fetching a checkpoint record? */
- bool randAccess;
-} XLogPageReadPrivate;
-
/*
* These variables track when we last obtained some WAL data to process,
* and where we got it from. (XLogReceiptSource is initially the same as
@@ -920,8 +913,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
@@ -1234,8 +1227,7 @@ XLogInsertRecord(XLogRecData *rdata,
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(), NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
@@ -4369,15 +4361,10 @@ CleanupBackupHistory(void)
* record is available.
*/
static XLogRecord *
-ReadRecord(XLogReaderState *xlogreader, int emode,
- bool fetching_ckpt)
+ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt)
{
XLogRecord *record;
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
-
- private->fetching_ckpt = fetching_ckpt;
- private->emode = emode;
- private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
+ bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
/* This is the first attempt to read this page. */
lastSourceFailed = false;
@@ -4385,8 +4372,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
for (;;)
{
char *errormsg;
+ XLogReadRecordResult result;
+
+ while ((result = XLogReadRecord(xlogreader, &record, &errormsg))
+ == XLREAD_NEED_DATA)
+ {
+ if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess))
+ break;
+
+ }
- record = XLogReadRecord(xlogreader, &errormsg);
ReadRecPtr = xlogreader->ReadRecPtr;
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
@@ -6456,7 +6451,6 @@ StartupXLOG(void)
bool backupFromStandby = false;
DBState dbstate_at_startup;
XLogReaderState *xlogreader;
- XLogPageReadPrivate private;
bool promoted = false;
struct stat st;
@@ -6615,13 +6609,9 @@ StartupXLOG(void)
OwnLatch(&XLogCtl->recoveryWakeupLatch);
/* Set up XLOG reader facility */
- MemSet(&private, 0, sizeof(XLogPageReadPrivate));
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.page_read = &XLogPageRead,
- .segment_open = NULL,
- .segment_close = wal_segment_close),
- &private);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -12107,12 +12097,13 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *readBuf)
+XLogPageRead(XLogReaderState *xlogreader,
+ bool fetching_ckpt, int emode, bool randAccess)
{
- XLogPageReadPrivate *private =
- (XLogPageReadPrivate *) xlogreader->private_data;
- int emode = private->emode;
+ char *readBuf = xlogreader->readBuf;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ int reqLen = xlogreader->readLen;
+ XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12155,8 +12146,8 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- private->randAccess,
- private->fetching_ckpt,
+ randAccess,
+ fetching_ckpt,
targetRecPtr))
{
if (readFile >= 0)
@@ -12261,6 +12252,7 @@ retry:
goto next_record_is_invalid;
}
+ Assert(xlogreader->readPagePtr == targetPagePtr);
xlogreader->readLen = readLen;
return true;
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f2345ab09e..661863e94b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -40,7 +40,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
+ XLogRecPtr PrevRecPtr, XLogRecord *record);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
static void ResetDecoder(XLogReaderState *state);
@@ -73,7 +73,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
*/
XLogReaderState *
XLogReaderAllocate(int wal_segment_size, const char *waldir,
- XLogReaderRoutine *routine, void *private_data)
+ WALSegmentCleanupCB cleanup_cb)
{
XLogReaderState *state;
@@ -84,7 +84,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
return NULL;
/* initialize caller-provided support functions */
- state->routine = *routine;
+ state->cleanup_cb = cleanup_cb;
state->max_block_id = -1;
@@ -107,8 +107,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* system_identifier initialized to zeroes above */
- state->private_data = private_data;
/* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
@@ -140,8 +138,8 @@ XLogReaderFree(XLogReaderState *state)
{
int block_id;
- if (state->seg.ws_file != -1)
- state->routine.segment_close(state);
+ if (state->seg.ws_file >= 0)
+ state->cleanup_cb(state);
for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++)
{
@@ -246,6 +244,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
/* Begin at the passed-in record pointer. */
state->EndRecPtr = RecPtr;
state->ReadRecPtr = InvalidXLogRecPtr;
+ state->readRecordState = XLREAD_NEXT_RECORD;
}
/*
@@ -254,12 +253,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* XLogBeginRead() or XLogFindNextRecord() must be called before the first call
* to XLogReadRecord().
*
- * If the page_read callback fails to read the requested data, NULL is
- * returned. The callback is expected to have reported the error; errormsg
- * is set to NULL.
+ * This function may return XLREAD_NEED_DATA several times before returning a
+ * result record. The caller shall read in some new data then call this
+ * function again with the same parameters.
*
- * If the reading fails for some other reason, NULL is also returned, and
- * *errormsg is set to a string with details of the failure.
+ * When a record is successfully read, returns XLREAD_SUCCESS with result
+ * record being stored in *record. Otherwise *record is NULL.
*
* Returns XLREAD_NEED_DATA if more data is needed to finish reading the
* current record. In that case, state->readPagePtr and state->readLen inform
@@ -304,307 +303,410 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* state. This behavior allows to continue reading a reacord switching among
* different souces, while streaming replication.
*/
-XLogRecord *
-XLogReadRecord(XLogReaderState *state, char **errormsg)
+XLogReadRecordResult
+XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
{
- XLogRecPtr RecPtr;
- XLogRecord *record;
- XLogRecPtr targetPagePtr;
- bool randAccess;
- uint32 len,
- total_len;
- uint32 targetRecOff;
- uint32 pageHeaderSize;
- bool gotheader;
+ XLogRecord *prec;
- /*
- * randAccess indicates whether to verify the previous-record pointer of
- * the record we're reading. We only do this if we're reading
- * sequentially, which is what we initially assume.
- */
- randAccess = false;
+ *record = NULL;
/* reset error state */
*errormsg = NULL;
state->errormsg_buf[0] = '\0';
- ResetDecoder(state);
-
- RecPtr = state->EndRecPtr;
-
- if (state->ReadRecPtr != InvalidXLogRecPtr)
- {
- /* read the record after the one we just read */
-
- /*
- * EndRecPtr is pointing to end+1 of the previous WAL record. If
- * we're at a page boundary, no more records can fit on the current
- * page. We must skip over the page header, but we can't do that until
- * we've read in the page, since the header size is variable.
- */
- }
- else
- {
- /*
- * Caller supplied a position to start at.
- *
- * In this case, EndRecPtr should already be pointing to a valid
- * record starting position.
- */
- Assert(XRecOffIsValid(RecPtr));
- randAccess = true;
- }
-
- state->currRecPtr = RecPtr;
-
- targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
- targetRecOff = RecPtr % XLOG_BLCKSZ;
-
- /*
- * Read the page containing the record into state->readBuf. Request enough
- * byte to cover the whole record header, or at least the part of it that
- * fits on the same page.
- */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
- targetRecOff != 0))
+ switch (state->readRecordState)
{
- if (!state->routine.page_read(state, state->readPagePtr, state->readLen,
- RecPtr, state->readBuf))
- break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /*
- * We have at least the page header, so we can examine it now.
- */
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- if (targetRecOff == 0)
- {
- /*
- * At page start, so skip over page header.
- */
- RecPtr += pageHeaderSize;
- targetRecOff = pageHeaderSize;
- }
- else if (targetRecOff < pageHeaderSize)
- {
- report_invalid_record(state, "invalid record offset at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ case XLREAD_NEXT_RECORD:
+ ResetDecoder(state);
- if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
- targetRecOff == pageHeaderSize)
- {
- report_invalid_record(state, "contrecord is requested by %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* XLogNeedData has verified the page header */
- Assert(pageHeaderSize <= state->readLen);
-
- /*
- * Read the record length.
- *
- * NB: Even though we use an XLogRecord pointer here, the whole record
- * header might not fit on this page. xl_tot_len is the first field of the
- * struct, so it must be on this page (the records are MAXALIGNed), but we
- * cannot access any other fields until we've verified that we got the
- * whole header.
- */
- record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ);
- total_len = record->xl_tot_len;
-
- /*
- * If the whole record header is on this page, validate it immediately.
- * Otherwise do just a basic sanity check on xl_tot_len, and validate the
- * rest of the header after reading it from the next page. The xl_tot_len
- * check is necessary here to ensure that we enter the "Need to reassemble
- * record" code path below; otherwise we might fail to apply
- * ValidXLogRecordHeader at all.
- */
- if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
- {
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record,
- randAccess))
- goto err;
- gotheader = true;
- }
- else
- {
- /* XXX: more validation should be done here */
- if (total_len < SizeOfXLogRecord)
- {
- report_invalid_record(state,
- "invalid record length at %X/%X: wanted %u, got %u",
- LSN_FORMAT_ARGS(RecPtr),
- (uint32) SizeOfXLogRecord, total_len);
- goto err;
- }
- gotheader = false;
- }
-
- len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ;
- if (total_len > len)
- {
- /* Need to reassemble record */
- char *contdata;
- XLogPageHeader pageHeader;
- char *buffer;
- uint32 gotlen;
-
- /*
- * Enlarge readRecordBuf as needed.
- */
- if (total_len > state->readRecordBufSize &&
- !allocate_recordbuf(state, total_len))
- {
- /* We treat this as a "bogus data" condition */
- report_invalid_record(state, "record length %u at %X/%X too long",
- total_len, LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
-
- /* Copy the first fragment of the record from the first page. */
- memcpy(state->readRecordBuf,
- state->readBuf + RecPtr % XLOG_BLCKSZ, len);
- buffer = state->readRecordBuf + len;
- gotlen = len;
-
- do
- {
- int rest_len = total_len - gotlen;
-
- /* Calculate pointer to beginning of next page */
- targetPagePtr += XLOG_BLCKSZ;
-
- /* Wait for the next page to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(rest_len, XLOG_BLCKSZ),
- false))
+ if (state->ReadRecPtr != InvalidXLogRecPtr)
+ {
+ /* read the record after the one we just read */
+
+ /*
+ * EndRecPtr is pointing to end+1 of the previous WAL record.
+ * If we're at a page boundary, no more records can fit on the
+ * current page. We must skip over the page header, but we
+ * can't do that until we've read in the page, since the
+ * header size is variable.
+ */
+ state->PrevRecPtr = state->ReadRecPtr;
+ state->ReadRecPtr = state->EndRecPtr;
+ }
+ else
+ {
+ /*
+ * Caller supplied a position to start at.
+ *
+ * In this case, EndRecPtr should already be pointing to a
+ * valid record starting position.
+ */
+ Assert(XRecOffIsValid(state->EndRecPtr));
+ state->ReadRecPtr = state->EndRecPtr;
+
+ /*
+ * We cannot verify the previous-record pointer when we're
+ * seeking to a particular record. Reset PrevRecPtr so that we
+ * won't try doing that.
+ */
+ state->PrevRecPtr = InvalidXLogRecPtr;
+ state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */
+ }
+
+ state->record_verified = false;
+ state->readRecordState = XLREAD_TOT_LEN;
+ /* fall through */
+
+ case XLREAD_TOT_LEN:
+ {
+ uint32 total_len;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+ XLogPageHeader pageHeader;
+
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ /*
+ * Check if we have enough data. For the first record in the
+ * page, the requesting length doesn't contain page header.
+ */
+ if (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ /* examine page header now. */
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ if (targetRecOff == 0)
+ {
+ /* At page start, so skip over page header. */
+ state->ReadRecPtr += pageHeaderSize;
+ targetRecOff = pageHeaderSize;
+ }
+ else if (targetRecOff < pageHeaderSize)
+ {
+ report_invalid_record(state, "invalid record offset at %X/%X",
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
+
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) &&
+ targetRecOff == pageHeaderSize)
+ {
+ report_invalid_record(state, "contrecord is requested by %X/%X",
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
+
+ /*
+ * Read the record length.
+ *
+ * NB: Even though we use an XLogRecord pointer here, the
+ * whole record header might not fit on this page. xl_tot_len
+ * is the first field of the struct, so it must be on this
+ * page (the records are MAXALIGNed), but we cannot access any
+ * other fields until we've verified that we got the whole
+ * header.
+ */
+ prec = (XLogRecord *) (state->readBuf +
+ state->ReadRecPtr % XLOG_BLCKSZ);
+ total_len = prec->xl_tot_len;
+
+ /*
+ * If the whole record header is on this page, validate it
+ * immediately. Otherwise do just a basic sanity check on
+ * xl_tot_len, and validate the rest of the header after
+ * reading it from the next page. The xl_tot_len check is
+ * necessary here to ensure that we enter the
+ * XLREAD_CONTINUATION state below; otherwise we might fail to
+ * apply ValidXLogRecordHeader at all.
+ */
+ if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+ else
+ {
+ /* XXX: more validation should be done here */
+ if (total_len < SizeOfXLogRecord)
+ {
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %u, got %u",
+ LSN_FORMAT_ARGS(state->ReadRecPtr),
+ (uint32) SizeOfXLogRecord, total_len);
+ goto err;
+ }
+ }
+
+ /*
+ * Wait for the rest of the record, or the part of the record
+ * that fit on the first page if crossed a page boundary, to
+ * become available.
+ */
+ state->recordGotLen = 0;
+ state->recordRemainLen = total_len;
+ state->readRecordState = XLREAD_FIRST_FRAGMENT;
+ }
+ /* fall through */
+
+ case XLREAD_FIRST_FRAGMENT:
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr,
- state->readBuf))
+ uint32 total_len = state->recordRemainLen;
+ uint32 request_len;
+ uint32 record_len;
+ XLogRecPtr targetPagePtr;
+ uint32 targetRecOff;
+
+ /*
+ * Wait for the rest of the record on the first page to become
+ * available
+ */
+ targetPagePtr =
+ state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
+ targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+
+ request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+ record_len = request_len - targetRecOff;
+
+ /* ReadRecPtr contains page header */
+ Assert(targetRecOff != 0);
+ if (XLogNeedData(state, targetPagePtr, request_len, true))
+ return XLREAD_NEED_DATA;
+
+ /* error out if caller supplied bogus page */
+ if (!state->page_verified)
+ goto err;
+
+ prec = (XLogRecord *) (state->readBuf + targetRecOff);
+
+ /* validate record header if not yet */
+ if (!state->record_verified && record_len >= SizeOfXLogRecord)
+ {
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr, prec))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+
+ if (total_len == record_len)
+ {
+ /* Record does not cross a page boundary */
+ Assert(state->record_verified);
+
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
+ goto err;
+
+ state->record_verified = true; /* to be tidy */
+
+ /* We already checked the header earlier */
+ state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
+ }
- if (!state->page_verified)
- goto err;
+ /*
+ * The record continues on the next page. Need to reassemble
+ * record
+ */
+ Assert(total_len > record_len);
- Assert(SizeOfXLogShortPHD <= state->readLen);
+ /* Enlarge readRecordBuf as needed. */
+ if (total_len > state->readRecordBufSize &&
+ !allocate_recordbuf(state, total_len))
+ {
+ /* We treat this as a "bogus data" condition */
+ report_invalid_record(state,
+ "record length %u at %X/%X too long",
+ total_len,
+ LSN_FORMAT_ARGS(state->ReadRecPtr));
+ goto err;
+ }
- /* Check that the continuation on next page looks valid */
- pageHeader = (XLogPageHeader) state->readBuf;
- if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
- {
- report_invalid_record(state,
- "there is no contrecord flag at %X/%X",
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
+ /* Copy the first fragment of the record from the first page. */
+ memcpy(state->readRecordBuf, state->readBuf + targetRecOff,
+ record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* Calculate pointer to beginning of next page */
+ state->recordContRecPtr = state->ReadRecPtr + record_len;
+ Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+
+ state->readRecordState = XLREAD_CONTINUATION;
}
+ /* fall through */
- /*
- * Cross-check that xlp_rem_len agrees with how much of the record
- * we expect there to be left.
- */
- if (pageHeader->xlp_rem_len == 0 ||
- total_len != (pageHeader->xlp_rem_len + gotlen))
+ case XLREAD_CONTINUATION:
{
- report_invalid_record(state,
- "invalid contrecord length %u (expected %lld) at %X/%X",
- pageHeader->xlp_rem_len,
- ((long long) total_len) - gotlen,
- LSN_FORMAT_ARGS(RecPtr));
- goto err;
- }
+ XLogPageHeader pageHeader;
+ uint32 pageHeaderSize;
+ XLogRecPtr targetPagePtr;
- /* Append the continuation from this page to the buffer */
- pageHeaderSize = XLogPageHeaderSize(pageHeader);
+ /*
+ * we enter this state only if we haven't read the whole
+ * record.
+ */
+ Assert(state->recordRemainLen > 0);
- Assert(pageHeaderSize <= state->readLen);
+ while (state->recordRemainLen > 0)
+ {
+ char *contdata;
+ uint32 request_len;
+ uint32 record_len;
- contdata = (char *) state->readBuf + pageHeaderSize;
- len = XLOG_BLCKSZ - pageHeaderSize;
- if (pageHeader->xlp_rem_len < len)
- len = pageHeader->xlp_rem_len;
+ /* Wait for the next page to become available */
+ targetPagePtr = state->recordContRecPtr;
- Assert(pageHeaderSize + len <= state->readLen);
- memcpy(buffer, (char *) contdata, len);
- buffer += len;
- gotlen += len;
+ /* this request contains page header */
+ Assert(targetPagePtr != 0);
+ if (XLogNeedData(state, targetPagePtr,
+ Min(state->recordRemainLen, XLOG_BLCKSZ),
+ false))
+ return XLREAD_NEED_DATA;
- /* If we just reassembled the record header, validate it. */
- if (!gotheader)
- {
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr,
- record, randAccess))
+ if (!state->page_verified)
+ goto err;
+
+ Assert(SizeOfXLogShortPHD <= state->readLen);
+
+ /* Check that the continuation on next page looks valid */
+ pageHeader = (XLogPageHeader) state->readBuf;
+ if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
+ {
+ report_invalid_record(
+ state,
+ "there is no contrecord flag at %X/%X reading %X/%X",
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr);
+ goto err;
+ }
+
+ /*
+ * Cross-check that xlp_rem_len agrees with how much of
+ * the record we expect there to be left.
+ */
+ if (pageHeader->xlp_rem_len == 0 ||
+ pageHeader->xlp_rem_len != state->recordRemainLen)
+ {
+ report_invalid_record(
+ state,
+ "invalid contrecord length %u at %X/%X reading %X/%X, expected %u",
+ pageHeader->xlp_rem_len,
+ (uint32) (state->recordContRecPtr >> 32),
+ (uint32) state->recordContRecPtr,
+ (uint32) (state->ReadRecPtr >> 32),
+ (uint32) state->ReadRecPtr,
+ state->recordRemainLen);
+ goto err;
+ }
+
+ /* Append the continuation from this page to the buffer */
+ pageHeaderSize = XLogPageHeaderSize(pageHeader);
+
+ /*
+ * XLogNeedData should have ensured that the whole page
+ * header was read
+ */
+ Assert(state->readLen >= pageHeaderSize);
+
+ contdata = (char *) state->readBuf + pageHeaderSize;
+ record_len = XLOG_BLCKSZ - pageHeaderSize;
+ if (pageHeader->xlp_rem_len < record_len)
+ record_len = pageHeader->xlp_rem_len;
+
+ request_len = record_len + pageHeaderSize;
+
+ /*
+ * XLogNeedData should have ensured all needed data was
+ * read
+ */
+ Assert(state->readLen >= request_len);
+
+ memcpy(state->readRecordBuf + state->recordGotLen,
+ (char *) contdata, record_len);
+ state->recordGotLen += record_len;
+ state->recordRemainLen -= record_len;
+
+ /* If we just reassembled the record header, validate it. */
+ if (!state->record_verified)
+ {
+ Assert(state->recordGotLen >= SizeOfXLogRecord);
+ if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
+ state->PrevRecPtr,
+ (XLogRecord *) state->readRecordBuf))
+ goto err;
+
+ state->record_verified = true;
+ }
+
+ /*
+ * Calculate pointer to beginning of next page, and
+ * continue
+ */
+ state->recordContRecPtr += XLOG_BLCKSZ;
+ }
+
+ /* targetPagePtr is pointing the last-read page here */
+ prec = (XLogRecord *) state->readRecordBuf;
+ if (!ValidXLogRecord(state, prec, state->ReadRecPtr))
goto err;
- gotheader = true;
- }
- } while (gotlen < total_len);
-
- Assert(gotheader);
-
- record = (XLogRecord *) state->readRecordBuf;
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- state->ReadRecPtr = RecPtr;
- state->EndRecPtr = targetPagePtr + pageHeaderSize
- + MAXALIGN(pageHeader->xlp_rem_len);
- }
- else
- {
- /* Wait for the record data to become available */
- while (XLogNeedData(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
- {
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+
+ pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+ state->EndRecPtr = targetPagePtr + pageHeaderSize
+ + MAXALIGN(pageHeader->xlp_rem_len);
+
+ *record = prec;
+ state->readRecordState = XLREAD_NEXT_RECORD;
break;
- }
-
- if (!state->page_verified)
- goto err;
-
- /* Record does not cross a page boundary */
- if (!ValidXLogRecord(state, record, RecPtr))
- goto err;
-
- state->EndRecPtr = RecPtr + MAXALIGN(total_len);
-
- state->ReadRecPtr = RecPtr;
+ }
}
/*
* Special processing if it's an XLOG SWITCH record
*/
- if (record->xl_rmid == RM_XLOG_ID &&
- (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
+ if ((*record)->xl_rmid == RM_XLOG_ID &&
+ ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
state->EndRecPtr += state->segcxt.ws_segsize - 1;
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- if (DecodeXLogRecord(state, record, errormsg))
- return record;
- else
- return NULL;
+ Assert(!*record || state->readLen >= 0);
+ if (DecodeXLogRecord(state, *record, errormsg))
+ return XLREAD_SUCCESS;
+
+ *record = NULL;
+ return XLREAD_FAIL;
err:
/*
- * Invalidate the read state. We might read from a different source after
+ * Invalidate the read page. We might read from a different source after
* failure.
*/
XLogReaderInvalReadState(state);
@@ -612,7 +714,8 @@ err:
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
- return NULL;
+ *record = NULL;
+ return XLREAD_FAIL;
}
/*
@@ -764,11 +867,12 @@ XLogReaderInvalReadState(XLogReaderState *state)
*
* This is just a convenience subroutine to avoid duplicated code in
* XLogReadRecord. It's not intended for use from anywhere else.
+ *
+ * If PrevRecPtr is valid, the xl_prev is is cross-checked with it.
*/
static bool
ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogRecPtr PrevRecPtr, XLogRecord *record,
- bool randAccess)
+ XLogRecPtr PrevRecPtr, XLogRecord *record)
{
if (record->xl_tot_len < SizeOfXLogRecord)
{
@@ -785,7 +889,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
record->xl_rmid, LSN_FORMAT_ARGS(RecPtr));
return false;
}
- if (randAccess)
+ if (PrevRecPtr == InvalidXLogRecPtr)
{
/*
* We can't exactly verify the prev-link, but surely it should be less
@@ -1015,11 +1119,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* XLogReadRecord() will read the next valid record.
*/
XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
+XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private)
{
XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
+ XLogRecord *record;
+ XLogReadRecordResult result;
char *errormsg;
Assert(!XLogRecPtrIsInvalid(RecPtr));
@@ -1052,9 +1159,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
while (XLogNeedData(state, targetPagePtr, targetRecOff,
targetRecOff != 0))
{
- if (!state->routine.page_read(state, state->readPagePtr,
- state->readLen,
- state->ReadRecPtr, state->readBuf))
+ if (!read_page(state, private))
break;
}
@@ -1106,8 +1211,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* or we just jumped over the remaining data of a continuation.
*/
XLogBeginRead(state, tmpRecPtr);
- while (XLogReadRecord(state, &errormsg) != NULL)
+ while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLREAD_FAIL)
{
+ if (result == XLREAD_NEED_DATA)
+ {
+ if (!read_page(state, private))
+ goto err;
+ continue;
+ }
+
/* past the record we've found, break out */
if (RecPtr <= state->ReadRecPtr)
{
@@ -1127,9 +1240,9 @@ err:
#endif /* FRONTEND */
/*
- * Helper function to ease writing of XLogRoutine->page_read callbacks.
- * If this function is used, caller must supply a segment_open callback in
- * 'state', as that is used here.
+ * Helper function to ease writing of page_read callback.
+ * If this function is used, caller must supply a segment_open callback and
+ * segment_close callback as that is used here.
*
* Read 'count' bytes into 'buf', starting at location 'startptr', from WAL
* fetched from timeline 'tli'.
@@ -1142,6 +1255,7 @@ err:
*/
bool
WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn,
char *buf, XLogRecPtr startptr, Size count, TimeLineID tli,
WALReadError *errinfo)
{
@@ -1173,10 +1287,10 @@ WALRead(XLogReaderState *state,
XLogSegNo nextSegNo;
if (state->seg.ws_file >= 0)
- state->routine.segment_close(state);
+ segclosefn(state);
XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize);
- state->routine.segment_open(state, nextSegNo, &tli);
+ segopenfn(state, nextSegNo, &tli);
/* This shouldn't happen -- indicates a bug in segment_open */
Assert(state->seg.ws_file >= 0);
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 46eda33f25..b003990745 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->readLen;
+ const XLogRecPtr lastReadPage = state->readPagePtr;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* current TLI has since become historical.
*/
if (lastReadPage == wantPage &&
- state->readLen != 0 &&
+ state->page_verified &&
lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1))
return;
@@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
char path[MAXPGPATH];
XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path);
state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
if (state->seg.ws_file >= 0)
return;
@@ -825,9 +825,11 @@ wal_segment_close(XLogReaderState *state)
* loop for now.
*/
bool
-read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
+read_local_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
TimeLineID tli;
@@ -940,11 +942,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli,
- &errinfo))
+ if (!WALRead(state, wal_segment_open, wal_segment_close,
+ cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo))
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
+ state->readPagePtr = targetPagePtr;
state->readLen = count;
return true;
}
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 2f6803637b..4f6e87f18d 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options,
TransactionId xmin_horizon,
bool need_full_snapshot,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options,
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
+ ctx->page_read = page_read;
ctx->reorder = ReorderBufferAllocate();
ctx->snapshot_builder =
@@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin,
ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon,
need_full_snapshot, false,
- xl_routine, prepare_write, do_write,
+ page_read, cleanup_cb, prepare_write, do_write,
update_progress);
/* call output plugin initialization callback */
@@ -476,7 +479,8 @@ LogicalDecodingContext *
CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress)
@@ -528,8 +532,8 @@ CreateDecodingContext(XLogRecPtr start_lsn,
ctx = StartupDecodingContext(output_plugin_options,
start_lsn, InvalidTransactionId, false,
- fast_forward, xl_routine, prepare_write,
- do_write, update_progress);
+ fast_forward, page_read, cleanup_cb,
+ prepare_write, do_write, update_progress);
/* call output plugin initialization callback */
old_context = MemoryContextSwitchTo(ctx->context);
@@ -585,7 +589,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx)
char *err = NULL;
/* the read_page callback waits for new WAL */
- record = XLogReadRecord(ctx->reader, &err);
+ while (XLogReadRecord(ctx->reader, &record, &err) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (err)
elog(ERROR, "%s", err);
if (!record)
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 01d354829b..8f8c129620 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
false,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
LogicalOutputPrepareWrite,
LogicalOutputWrite, NULL);
@@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
XLogRecord *record;
char *errm = NULL;
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index d9d36879ed..7ab0b804e4 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin,
ctx = CreateInitDecodingContext(plugin, NIL,
false, /* just catalogs is OK */
restart_lsn,
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
ctx = CreateDecodingContext(InvalidXLogRecPtr,
NIL,
true, /* fast_forward */
- XL_ROUTINE(.page_read = read_local_xlog_page,
- .segment_open = wal_segment_open,
- .segment_close = wal_segment_close),
+ read_local_xlog_page,
+ wal_segment_close,
NULL, NULL, NULL);
/*
@@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
* Read records. No changes are generated in fast_forward mode,
* but snapbuilder/slot statuses are updated properly.
*/
- record = XLogReadRecord(ctx->reader, &errm);
+ while (XLogReadRecord(ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!ctx->page_read(ctx->reader))
+ break;
+ }
+
if (errm)
elog(ERROR, "%s", errm);
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index a4d6f30957..b024bbc3cd 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -580,10 +580,7 @@ StartReplication(StartReplicationCmd *cmd)
/* create xlogreader for physical replication */
xlogreader =
- XLogReaderAllocate(wal_segment_size, NULL,
- XL_ROUTINE(.segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
- NULL);
+ XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
if (!xlogreader)
ereport(ERROR,
@@ -807,9 +804,11 @@ StartReplication(StartReplicationCmd *cmd)
* set every time WAL is flushed.
*/
static bool
-logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page)
+logical_read_xlog_page(XLogReaderState *state)
{
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
WALReadError errinfo;
@@ -837,7 +836,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
count = flushptr - targetPagePtr; /* part of the page available */
/* now actually read the data, we know it's there */
- if (!WALRead(state,
+ if (!WALRead(state, WalSndSegmentOpen, wal_segment_close,
cur_page,
targetPagePtr,
XLOG_BLCKSZ,
@@ -1011,9 +1010,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot,
InvalidXLogRecPtr,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
@@ -1171,9 +1169,8 @@ StartLogicalReplication(StartReplicationCmd *cmd)
*/
logical_decoding_ctx =
CreateDecodingContext(cmd->startpoint, cmd->options, false,
- XL_ROUTINE(.page_read = logical_read_xlog_page,
- .segment_open = WalSndSegmentOpen,
- .segment_close = wal_segment_close),
+ logical_read_xlog_page,
+ wal_segment_close,
WalSndPrepareWrite, WalSndWriteData,
WalSndUpdateProgress);
xlogreader = logical_decoding_ctx->reader;
@@ -2749,7 +2746,7 @@ XLogSendPhysical(void)
enlargeStringInfo(&output_message, nbytes);
retry:
- if (!WALRead(xlogreader,
+ if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close,
&output_message.data[output_message.len],
startptr,
nbytes,
@@ -2847,7 +2844,12 @@ XLogSendLogical(void)
*/
WalSndCaughtUp = false;
- record = XLogReadRecord(logical_decoding_ctx->reader, &errm);
+ while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader))
+ break;
+ }
/* xlog record was invalid */
if (errm != NULL)
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cf119848b0..712c85281c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -41,15 +41,9 @@ static int xlogreadfd = -1;
static XLogSegNo xlogreadsegno = -1;
static char xlogfpath[MAXPGPATH];
-typedef struct XLogPageReadPrivate
-{
- const char *restoreCommand;
- int tliIndex;
-} XLogPageReadPrivate;
-
-static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
+ const char *datadir, int *tliIndex,
+ const char *restoreCommand);
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
@@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, startpoint);
do
{
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
XLogRecord *record;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
XLogRecPtr endptr;
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
XLogBeginRead(xlogreader, ptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
if (errormsg)
@@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr searchptr;
XLogReaderState *xlogreader;
char *errormsg;
- XLogPageReadPrivate private;
/*
* The given fork pointer points to the end of the last common record,
@@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
forkptr += SizeOfXLogShortPHD;
}
- private.tliIndex = tliIndex;
- private.restoreCommand = restoreCommand;
- xlogreader = XLogReaderAllocate(WalSegSz, datadir,
- XL_ROUTINE(.page_read = &SimpleXLogPageRead),
- &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
if (xlogreader == NULL)
pg_fatal("out of memory");
@@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
uint8 info;
XLogBeginRead(xlogreader, searchptr);
- record = XLogReadRecord(xlogreader, &errormsg);
+ while (XLogReadRecord(xlogreader, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!SimpleXLogPageRead(xlogreader, datadir,
+ &tliIndex, restoreCommand))
+ break;
+ }
if (record == NULL)
{
@@ -247,10 +244,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
/* XLogReader callback function, to read a WAL page */
static bool
-SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
+SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
+ int *tliIndex, const char *restoreCommand)
{
- XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
+ XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
+ char *readBuf = xlogreader->readBuf;
uint32 targetPageOff;
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
@@ -283,14 +281,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
* be done both forward and backward, consider also switching timeline
* accordingly.
*/
- while (private->tliIndex < targetNentries - 1 &&
- targetHistory[private->tliIndex].end < targetSegEnd)
- private->tliIndex++;
- while (private->tliIndex > 0 &&
- targetHistory[private->tliIndex].begin >= targetSegEnd)
- private->tliIndex--;
+ while (*tliIndex < targetNentries - 1 &&
+ targetHistory[*tliIndex].end < targetSegEnd)
+ (*tliIndex)++;
+ while (*tliIndex > 0 &&
+ targetHistory[*tliIndex].begin >= targetSegEnd)
+ (*tliIndex)--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ XLogFileName(xlogfname, targetHistory[*tliIndex].tli,
xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
@@ -303,7 +301,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
/*
* If we have no restore_command to execute, then exit.
*/
- if (private->restoreCommand == NULL)
+ if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
xlogreader->readLen = -1;
@@ -317,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
xlogfname,
WalSegSz,
- private->restoreCommand);
+ restoreCommand);
if (xlogreadfd < 0)
{
@@ -359,7 +357,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
Assert(targetSegNo == xlogreadsegno);
- xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
+ xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
xlogreader->readLen = XLOG_BLCKSZ;
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 75ece5c658..c4047b92b5 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -330,12 +330,17 @@ WALDumpCloseSegment(XLogReaderState *state)
state->seg.ws_file = -1;
}
-/* pg_waldump's XLogReaderRoutine->page_read callback */
+/*
+ * pg_waldump's WAL page rader, also used as page_read callback for
+ * XLogFindNextRecord
+ */
static bool
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+WALDumpReadPage(XLogReaderState *state, void *priv)
{
- XLogDumpPrivate *private = state->private_data;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ char *readBuff = state->readBuf;
+ XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -353,8 +358,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
}
}
- if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline,
- &errinfo))
+ if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
+ readBuff, targetPagePtr, count, private->timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -374,6 +379,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
+ Assert(count >= state->readLen);
state->readLen = count;
return true;
}
@@ -1044,16 +1050,14 @@ main(int argc, char **argv)
/* we have everything we need, start reading */
xlogreader_state =
- XLogReaderAllocate(WalSegSz, waldir,
- XL_ROUTINE(.page_read = WALDumpReadPage,
- .segment_open = WALDumpOpenSegment,
- .segment_close = WALDumpCloseSegment),
- &private);
+ XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+
if (!xlogreader_state)
fatal_error("out of memory");
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr);
+ first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
+ &WALDumpReadPage, (void*) &private);
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
@@ -1076,7 +1080,13 @@ main(int argc, char **argv)
for (;;)
{
/* try to read the next record */
- record = XLogReadRecord(xlogreader_state, &errormsg);
+ while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
+ XLREAD_NEED_DATA)
+ {
+ if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ break;
+ }
+
if (!record)
{
if (!config.follow || private.endptr_reached)
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 5d9e0d3292..1492f1992d 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -57,64 +57,15 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
-/* Function type definition for the read_page callback */
-typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
- XLogRecPtr targetPagePtr,
- int reqLen,
- XLogRecPtr targetRecPtr,
- char *readBuf);
+/* Function type definition for the segment cleanup callback */
+typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
+
+/* Function type definition for the open/close callbacks for WALRead() */
typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader);
-typedef struct XLogReaderRoutine
-{
- /*
- * Data input callback
- *
- * This callback shall read at least reqLen valid bytes of the xlog page
- * starting at targetPagePtr, and store them in readBuf. The callback
- * shall return the number of bytes read (never more than XLOG_BLCKSZ), or
- * -1 on failure. The callback shall sleep, if necessary, to wait for the
- * requested bytes to become available. The callback will not be invoked
- * again for the same page unless more than the returned number of bytes
- * are needed.
- *
- * targetRecPtr is the position of the WAL record we're reading. Usually
- * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs
- * to read and verify the page or segment header, before it reads the
- * actual WAL record it's interested in. In that case, targetRecPtr can
- * be used to determine which timeline to read the page from.
- *
- * The callback shall set ->seg.ws_tli to the TLI of the file the page was
- * read from.
- */
- XLogPageReadCB page_read;
-
- /*
- * Callback to open the specified WAL segment for reading. ->seg.ws_file
- * shall be set to the file descriptor of the opened segment. In case of
- * failure, an error shall be raised by the callback and it shall not
- * return.
- *
- * "nextSegNo" is the number of the segment to be opened.
- *
- * "tli_p" is an input/output argument. WALRead() uses it to pass the
- * timeline in which the new segment should be found, but the callback can
- * use it to return the TLI that it actually opened.
- */
- WALSegmentOpenCB segment_open;
-
- /*
- * WAL segment close callback. ->seg.ws_file shall be set to a negative
- * number.
- */
- WALSegmentCloseCB segment_close;
-} XLogReaderRoutine;
-
-#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__}
-
typedef struct
{
/* Is this block ref in use? */
@@ -144,12 +95,36 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Return code from XLogReadRecord */
+typedef enum XLogReadRecordResult
+{
+ XLREAD_SUCCESS, /* record is successfully read */
+ XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */
+ XLREAD_FAIL /* failed during reading a record */
+} XLogReadRecordResult;
+
+/*
+ * internal state of XLogReadRecord
+ *
+ * XLogReadState runs a state machine while reading a record. Theses states
+ * are not seen outside the function. Each state may repeat several times
+ * exiting requesting caller for new data. See the comment of XLogReadRecrod
+ * for details.
+ */
+typedef enum XLogReadRecordState
+{
+ XLREAD_NEXT_RECORD,
+ XLREAD_TOT_LEN,
+ XLREAD_FIRST_FRAGMENT,
+ XLREAD_CONTINUATION
+} XLogReadRecordState;
+
struct XLogReaderState
{
/*
* Operational callbacks
*/
- XLogReaderRoutine routine;
+ WALSegmentCleanupCB cleanup_cb;
/* ----------------------------------------
* Public parameters
@@ -162,18 +137,14 @@ struct XLogReaderState
*/
uint64 system_identifier;
- /*
- * Opaque data for callbacks to use. Not used by XLogReader.
- */
- void *private_data;
-
/*
* Start and end point of last record read. EndRecPtr is also used as the
* position to read next. Calling XLogBeginRead() sets EndRecPtr to the
* starting position and ReadRecPtr to invalid.
*/
- XLogRecPtr ReadRecPtr; /* start of last record read */
+ XLogRecPtr ReadRecPtr; /* start of last record read or being read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ XLogRecPtr PrevRecPtr; /* start of previous record read */
/* ----------------------------------------
* Communication with page reader
@@ -186,7 +157,9 @@ struct XLogReaderState
* read by reader, which must be larger than
* the request, or -1 on error */
char *readBuf; /* buffer to store data */
- bool page_verified; /* is the page on the buffer verified? */
+ bool page_verified; /* is the page header on the buffer verified? */
+ bool record_verified; /* is the current record header verified? */
+
/* ----------------------------------------
@@ -228,8 +201,6 @@ struct XLogReaderState
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
- /* beginning of the WAL record being read. */
- XLogRecPtr currRecPtr;
/* timeline to read it from, 0 if a lookup is required */
TimeLineID currTLI;
@@ -256,6 +227,15 @@ struct XLogReaderState
char *readRecordBuf;
uint32 readRecordBufSize;
+ /*
+ * XLogReadRecord() state
+ */
+ XLogReadRecordState readRecordState; /* state machine state */
+ int recordGotLen; /* amount of current record that has already
+ * been read */
+ int recordRemainLen; /* length of current record that remains */
+ XLogRecPtr recordContRecPtr; /* where the current record continues */
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -263,9 +243,7 @@ struct XLogReaderState
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
- XLogReaderRoutine *routine,
- void *private_data);
-extern XLogReaderRoutine *LocalXLogReaderRoutine(void);
+ WALSegmentCleanupCB cleanup_cb);
/* Free an XLogReader */
extern void XLogReaderFree(XLogReaderState *state);
@@ -273,12 +251,17 @@ extern void XLogReaderFree(XLogReaderState *state);
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
+/* Function type definition for the read_page callback */
+typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
+ void *private);
+extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
+ XLogFindNextRecordCB read_page, void *private);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
-extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
- char **errormsg);
+extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state,
+ XLogRecord **record,
+ char **errormsg);
/* Validate a page */
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
@@ -298,6 +281,7 @@ typedef struct WALReadError
} WALReadError;
extern bool WALRead(XLogReaderState *state,
+ WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn,
char *buf, XLogRecPtr startptr, Size count,
TimeLineID tli, WALReadError *errinfo);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 364a21c4ea..397fb27fc2 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern bool read_local_xlog_page(XLogReaderState *state,
- XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetRecPtr, char *cur_page);
+extern bool read_local_xlog_page(XLogReaderState *state);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index e28c990382..bd9a5c6c2b 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -384,7 +384,7 @@
* Enable debugging print statements for WAL-related operations; see
* also the wal_debug GUC var.
*/
-/* #define WAL_DEBUG */
+#define WAL_DEBUG
/*
* Enable tracing of resource consumption during sort operations;
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index af551d6f4e..94e278ef81 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC
TransactionId xid
);
+typedef struct LogicalDecodingContext LogicalDecodingContext;
+
+typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx);
+
typedef struct LogicalDecodingContext
{
/* memory context this is all allocated in */
@@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext
/* infrastructure pieces for decoding */
XLogReaderState *reader;
+ LogicalDecodingXLogPageReadCB page_read;
struct ReorderBuffer *reorder;
struct SnapBuild *snapshot_builder;
@@ -105,14 +110,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
List *output_plugin_options,
bool need_full_snapshot,
XLogRecPtr restart_lsn,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
List *output_plugin_options,
bool fast_forward,
- XLogReaderRoutine *xl_routine,
+ LogicalDecodingXLogPageReadCB page_read,
+ WALSegmentCleanupCB cleanup_cb,
LogicalOutputPluginWriterPrepareWrite prepare_write,
LogicalOutputPluginWriterWrite do_write,
LogicalOutputPluginWriterUpdateProgress update_progress);
--
2.27.0
From 64f2c5ae354fff355dc6c78807bc1f9e255e5956 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 10 Sep 2019 17:28:48 +0900
Subject: [PATCH v19 3/5] Remove globals readOff, readLen and readSegNo.
The first two global variables are duplicated in XLogReaderState.
Remove them, and also readSegNo, which should move into that struct too.
Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp>
Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp
---
src/backend/access/transam/xlog.c | 77 ++++++++++++++-----------------
1 file changed, 35 insertions(+), 42 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b7d7e6d31b..9a9835f05d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -811,17 +811,13 @@ static XLogSegNo openLogSegNo = 0;
* These variables are used similarly to the ones above, but for reading
* the XLOG. Note, however, that readOff generally represents the offset
* of the page just read, not the seek position of the FD itself, which
- * will be just past that page. readLen indicates how much of the current
- * page has been read into readBuf, and readSource indicates where we got
- * the currently open file from.
+ * will be just past that page. readSource indicates where we got the
+ * currently open file from.
* Note: we could use Reserve/ReleaseExternalFD to track consumption of
* this FD too; but it doesn't currently seem worthwhile, since the XLOG is
* not read by general-purpose sessions.
*/
static int readFile = -1;
-static XLogSegNo readSegNo = 0;
-static uint32 readOff = 0;
-static uint32 readLen = 0;
static XLogSource readSource = XLOG_FROM_ANY;
/*
@@ -913,10 +909,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static bool XLogPageRead(XLogReaderState *xlogreader,
+static bool XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr);
+ bool fetching_ckpt,
+ XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo);
static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
static void XLogFileClose(void);
static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -7808,7 +7806,8 @@ StartupXLOG(void)
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
+ Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) ==
+ XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
@@ -12097,13 +12096,14 @@ CancelBackup(void)
* sleep and retry.
*/
static bool
-XLogPageRead(XLogReaderState *xlogreader,
+XLogPageRead(XLogReaderState *state,
bool fetching_ckpt, int emode, bool randAccess)
{
- char *readBuf = xlogreader->readBuf;
- XLogRecPtr targetPagePtr = xlogreader->readPagePtr;
- int reqLen = xlogreader->readLen;
- XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr;
+ char *readBuf = state->readBuf;
+ XLogRecPtr targetPagePtr = state->readPagePtr;
+ int reqLen = state->readLen;
+ int readLen = 0;
+ XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
int r;
@@ -12116,7 +12116,7 @@ XLogPageRead(XLogReaderState *xlogreader,
* is not in the currently open one.
*/
if (readFile >= 0 &&
- !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
+ !XLByteInSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
@@ -12124,10 +12124,10 @@ XLogPageRead(XLogReaderState *xlogreader,
*/
if (bgwriterLaunched)
{
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->seg.ws_segno))
{
(void) GetRedoRecPtr();
- if (XLogCheckpointNeeded(readSegNo))
+ if (XLogCheckpointNeeded(state->seg.ws_segno))
RequestCheckpoint(CHECKPOINT_CAUSE_XLOG);
}
}
@@ -12137,7 +12137,7 @@ XLogPageRead(XLogReaderState *xlogreader,
readSource = XLOG_FROM_ANY;
}
- XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
+ XLByteToSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size);
retry:
/* See if we need to retrieve more data */
@@ -12146,17 +12146,14 @@ retry:
flushedUpto < targetPagePtr + reqLen))
{
if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen,
- randAccess,
- fetching_ckpt,
- targetRecPtr))
+ randAccess, fetching_ckpt,
+ targetRecPtr, state->seg.ws_segno))
{
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
-
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
}
@@ -12184,40 +12181,36 @@ retry:
else
readLen = XLOG_BLCKSZ;
- /* Read the requested page */
- readOff = targetPageOff;
-
pgstat_report_wait_start(WAIT_EVENT_WAL_READ);
- r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff);
+ r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff);
if (r != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
int save_errno = errno;
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
+ XLogFileName(fname, curFileTLI, state->seg.ws_segno, wal_segment_size);
if (r < 0)
{
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
- fname, readOff)));
+ fname, targetPageOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("could not read from log segment %s, offset %u: read %d of %zu",
- fname, readOff, r, (Size) XLOG_BLCKSZ)));
+ fname, targetPageOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
pgstat_report_wait_end();
- Assert(targetSegNo == readSegNo);
- Assert(targetPageOff == readOff);
+ Assert(targetSegNo == state->seg.ws_segno);
Assert(reqLen <= readLen);
- xlogreader->seg.ws_tli = curFileTLI;
+ state->seg.ws_tli = curFileTLI;
/*
* Check the page header immediately, so that we can retry immediately if
@@ -12245,15 +12238,15 @@ retry:
* Validating the page header is cheap enough that doing it twice
* shouldn't be a big deal from a performance point of view.
*/
- if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf))
+ if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf))
{
- /* reset any error XLogReaderValidatePageHeader() might have set */
- xlogreader->errormsg_buf[0] = '\0';
+ /* reset any error StateValidatePageHeader() might have set */
+ state->errormsg_buf[0] = '\0';
goto next_record_is_invalid;
}
- Assert(xlogreader->readPagePtr == targetPagePtr);
- xlogreader->readLen = readLen;
+ Assert(state->readPagePtr == targetPagePtr);
+ state->readLen = readLen;
return true;
next_record_is_invalid:
@@ -12262,14 +12255,13 @@ next_record_is_invalid:
if (readFile >= 0)
close(readFile);
readFile = -1;
- readLen = 0;
readSource = XLOG_FROM_ANY;
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- xlogreader->readLen = -1;
+ state->readLen = -1;
return false;
}
@@ -12301,7 +12293,8 @@ next_record_is_invalid:
*/
static bool
WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
- bool fetching_ckpt, XLogRecPtr tliRecPtr)
+ bool fetching_ckpt, XLogRecPtr tliRecPtr,
+ XLogSegNo readSegNo)
{
static TimestampTz last_fail_time = 0;
TimestampTz now;
--
2.27.0
From 3ef3ec9d6864464f6bf9a248222d26905d932e6c Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Wed, 7 Apr 2021 15:32:10 +0900
Subject: [PATCH v19 4/5] Make XLogFindNextRecord not use callback function
The last function that uses page-read callback is
XLogFindNextRecord. Lets make it free from call-back. This also
simplifies the interface of WALDumpReadPage.
---
src/backend/access/transam/xlogreader.c | 73 ++++++++-------
src/bin/pg_waldump/pg_waldump.c | 115 ++++++++++++------------
src/include/access/xlogreader.h | 12 ++-
3 files changed, 110 insertions(+), 90 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 661863e94b..89c59843b9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1107,6 +1107,22 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* here.
*/
+XLogFindNextRecordState *
+InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr)
+{
+ XLogFindNextRecordState *state = (XLogFindNextRecordState *)
+ palloc_extended(sizeof(XLogFindNextRecordState),
+ MCXT_ALLOC_NO_OOM | MCXT_ALLOC_ZERO);
+ if (!state)
+ return NULL;
+
+ state->reader_state = reader_state;
+ state->targetRecPtr = start_ptr;
+ state->currRecPtr = start_ptr;
+
+ return state;
+}
+
/*
* Find the first record with an lsn >= RecPtr.
*
@@ -1118,24 +1134,21 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
* This positions the reader, like XLogBeginRead(), so that the next call to
* XLogReadRecord() will read the next valid record.
*/
-XLogRecPtr
-XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogFindNextRecordCB read_page, void *private)
+bool
+XLogFindNextRecord(XLogFindNextRecordState *state)
{
- XLogRecPtr tmpRecPtr;
XLogRecPtr found = InvalidXLogRecPtr;
XLogPageHeader header;
XLogRecord *record;
XLogReadRecordResult result;
char *errormsg;
- Assert(!XLogRecPtrIsInvalid(RecPtr));
+ Assert(!XLogRecPtrIsInvalid(state->currRecPtr));
/*
* skip over potential continuation data, keeping in mind that it may span
* multiple pages
*/
- tmpRecPtr = RecPtr;
while (true)
{
XLogRecPtr targetPagePtr;
@@ -1151,27 +1164,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* XLogNeedData() is prepared to handle that and will read at least
* short page-header worth of data
*/
- targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+ targetRecOff = state->currRecPtr % XLOG_BLCKSZ;
/* scroll back to page boundary */
- targetPagePtr = tmpRecPtr - targetRecOff;
+ targetPagePtr = state->currRecPtr - targetRecOff;
- while (XLogNeedData(state, targetPagePtr, targetRecOff,
+ if (XLogNeedData(state->reader_state, targetPagePtr, targetRecOff,
targetRecOff != 0))
- {
- if (!read_page(state, private))
- break;
- }
+ return true;
- if (!state->page_verified)
+ if (!state->reader_state->page_verified)
goto err;
- header = (XLogPageHeader) state->readBuf;
+ header = (XLogPageHeader) state->reader_state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
/* we should have read the page header */
- Assert(state->readLen >= pageHeaderSize);
+ Assert(state->reader_state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
@@ -1186,21 +1196,21 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* Note that record headers are MAXALIGN'ed
*/
if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
- tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+ state->currRecPtr = targetPagePtr + XLOG_BLCKSZ;
else
{
/*
* The previous continuation record ends in this page. Set
- * tmpRecPtr to point to the first valid record
+ * state->currRecPtr to point to the first valid record
*/
- tmpRecPtr = targetPagePtr + pageHeaderSize
+ state->currRecPtr = targetPagePtr + pageHeaderSize
+ MAXALIGN(header->xlp_rem_len);
break;
}
}
else
{
- tmpRecPtr = targetPagePtr + pageHeaderSize;
+ state->currRecPtr = targetPagePtr + pageHeaderSize;
break;
}
}
@@ -1210,31 +1220,28 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
* because either we're at the first record after the beginning of a page
* or we just jumped over the remaining data of a continuation.
*/
- XLogBeginRead(state, tmpRecPtr);
- while ((result = XLogReadRecord(state, &record, &errormsg)) !=
+ XLogBeginRead(state->reader_state, state->currRecPtr);
+ while ((result = XLogReadRecord(state->reader_state, &record, &errormsg)) !=
XLREAD_FAIL)
{
if (result == XLREAD_NEED_DATA)
- {
- if (!read_page(state, private))
- goto err;
- continue;
- }
+ return true;
/* past the record we've found, break out */
- if (RecPtr <= state->ReadRecPtr)
+ if (state->targetRecPtr <= state->reader_state->ReadRecPtr)
{
/* Rewind the reader to the beginning of the last record. */
- found = state->ReadRecPtr;
- XLogBeginRead(state, found);
- return found;
+ state->currRecPtr = state->reader_state->ReadRecPtr;
+ XLogBeginRead(state->reader_state, found);
+ return false;
}
}
err:
- XLogReaderInvalReadState(state);
+ XLogReaderInvalReadState(state->reader_state);
- return InvalidXLogRecPtr;
+ state->currRecPtr = InvalidXLogRecPtr;;
+ return false;
}
#endif /* FRONTEND */
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c4047b92b5..ab2d079bdb 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -29,14 +29,6 @@ static const char *progname;
static int WalSegSz;
-typedef struct XLogDumpPrivate
-{
- TimeLineID timeline;
- XLogRecPtr startptr;
- XLogRecPtr endptr;
- bool endptr_reached;
-} XLogDumpPrivate;
-
typedef struct XLogDumpConfig
{
/* display options */
@@ -331,35 +323,40 @@ WALDumpCloseSegment(XLogReaderState *state)
}
/*
- * pg_waldump's WAL page rader, also used as page_read callback for
- * XLogFindNextRecord
+ * pg_waldump's WAL page rader
+ *
+ * timeline and startptr specifies the LSN, and reads up to endptr.
*/
static bool
-WALDumpReadPage(XLogReaderState *state, void *priv)
+WALDumpReadPage(XLogReaderState *state, TimeLineID timeline,
+ XLogRecPtr startptr, XLogRecPtr endptr)
{
XLogRecPtr targetPagePtr = state->readPagePtr;
int reqLen = state->readLen;
char *readBuff = state->readBuf;
- XLogDumpPrivate *private = (XLogDumpPrivate *) priv;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
- if (private->endptr != InvalidXLogRecPtr)
+ /* determine the number of bytes to read on the page */
+ if (endptr != InvalidXLogRecPtr)
{
- if (targetPagePtr + XLOG_BLCKSZ <= private->endptr)
+ if (targetPagePtr + XLOG_BLCKSZ <= endptr)
count = XLOG_BLCKSZ;
- else if (targetPagePtr + reqLen <= private->endptr)
- count = private->endptr - targetPagePtr;
+ else if (targetPagePtr + reqLen <= endptr)
+ count = endptr - targetPagePtr;
else
{
- private->endptr_reached = true;
+ /* Notify xlogreader that we didn't read at all */
state->readLen = -1;
return false;
}
}
+ /* We should read more than requested by xlogreader */
+ Assert(count >= state->readLen);
+
if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment,
- readBuff, targetPagePtr, count, private->timeline, &errinfo))
+ readBuff, targetPagePtr, count, timeline, &errinfo))
{
WALOpenSegment *seg = &errinfo.wre_seg;
char fname[MAXPGPATH];
@@ -379,7 +376,7 @@ WALDumpReadPage(XLogReaderState *state, void *priv)
(Size) errinfo.wre_req);
}
- Assert(count >= state->readLen);
+ /* Notify xlogreader of how many bytes we have read */
state->readLen = count;
return true;
}
@@ -762,7 +759,10 @@ main(int argc, char **argv)
uint32 xlogid;
uint32 xrecoff;
XLogReaderState *xlogreader_state;
- XLogDumpPrivate private;
+ XLogFindNextRecordState *findnext_state;
+ TimeLineID timeline;
+ XLogRecPtr startptr;
+ XLogRecPtr endptr;
XLogDumpConfig config;
XLogDumpStats stats;
XLogRecord *record;
@@ -808,14 +808,9 @@ main(int argc, char **argv)
}
}
- memset(&private, 0, sizeof(XLogDumpPrivate));
- memset(&config, 0, sizeof(XLogDumpConfig));
- memset(&stats, 0, sizeof(XLogDumpStats));
-
- private.timeline = 1;
- private.startptr = InvalidXLogRecPtr;
- private.endptr = InvalidXLogRecPtr;
- private.endptr_reached = false;
+ timeline = 1;
+ startptr = InvalidXLogRecPtr;
+ endptr = InvalidXLogRecPtr;
config.quiet = false;
config.bkp_details = false;
@@ -849,7 +844,7 @@ main(int argc, char **argv)
optarg);
goto bad_argument;
}
- private.endptr = (uint64) xlogid << 32 | xrecoff;
+ endptr = (uint64) xlogid << 32 | xrecoff;
break;
case 'f':
config.follow = true;
@@ -902,10 +897,10 @@ main(int argc, char **argv)
goto bad_argument;
}
else
- private.startptr = (uint64) xlogid << 32 | xrecoff;
+ startptr = (uint64) xlogid << 32 | xrecoff;
break;
case 't':
- if (sscanf(optarg, "%d", &private.timeline) != 1)
+ if (sscanf(optarg, "%d", &timeline) != 1)
{
pg_log_error("could not parse timeline \"%s\"", optarg);
goto bad_argument;
@@ -982,21 +977,21 @@ main(int argc, char **argv)
close(fd);
/* parse position from file */
- XLogFromFileName(fname, &private.timeline, &segno, WalSegSz);
+ XLogFromFileName(fname, &timeline, &segno, WalSegSz);
- if (XLogRecPtrIsInvalid(private.startptr))
- XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, private.startptr);
- else if (!XLByteInSeg(private.startptr, segno, WalSegSz))
+ if (XLogRecPtrIsInvalid(startptr))
+ XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, startptr);
+ else if (!XLByteInSeg(startptr, segno, WalSegSz))
{
pg_log_error("start WAL location %X/%X is not inside file \"%s\"",
- LSN_FORMAT_ARGS(private.startptr),
+ LSN_FORMAT_ARGS(startptr),
fname);
goto bad_argument;
}
/* no second file specified, set end position */
- if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(private.endptr))
- XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, private.endptr);
+ if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(endptr))
+ XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, endptr);
/* parse ENDSEG if passed */
if (optind + 1 < argc)
@@ -1012,26 +1007,26 @@ main(int argc, char **argv)
close(fd);
/* parse position from file */
- XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz);
+ XLogFromFileName(fname, &timeline, &endsegno, WalSegSz);
if (endsegno < segno)
fatal_error("ENDSEG %s is before STARTSEG %s",
argv[optind + 1], argv[optind]);
- if (XLogRecPtrIsInvalid(private.endptr))
+ if (XLogRecPtrIsInvalid(endptr))
XLogSegNoOffsetToRecPtr(endsegno + 1, 0, WalSegSz,
- private.endptr);
+ endptr);
/* set segno to endsegno for check of --end */
segno = endsegno;
}
- if (!XLByteInSeg(private.endptr, segno, WalSegSz) &&
- private.endptr != (segno + 1) * WalSegSz)
+ if (!XLByteInSeg(endptr, segno, WalSegSz) &&
+ endptr != (segno + 1) * WalSegSz)
{
pg_log_error("end WAL location %X/%X is not inside file \"%s\"",
- LSN_FORMAT_ARGS(private.endptr),
+ LSN_FORMAT_ARGS(endptr),
argv[argc - 1]);
goto bad_argument;
}
@@ -1040,7 +1035,7 @@ main(int argc, char **argv)
waldir = identify_target_directory(waldir, NULL);
/* we don't know what to print */
- if (XLogRecPtrIsInvalid(private.startptr))
+ if (XLogRecPtrIsInvalid(startptr))
{
pg_log_error("no start WAL location given");
goto bad_argument;
@@ -1055,27 +1050,37 @@ main(int argc, char **argv)
if (!xlogreader_state)
fatal_error("out of memory");
+ findnext_state =
+ InitXLogFindNextRecord(xlogreader_state, startptr);
+
+ if (!findnext_state)
+ fatal_error("out of memory");
+
/* first find a valid recptr to start from */
- first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
- &WALDumpReadPage, (void*) &private);
+ while (XLogFindNextRecord(findnext_state))
+ {
+ if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr))
+ break;
+ }
+ first_record = findnext_state->currRecPtr;
if (first_record == InvalidXLogRecPtr)
fatal_error("could not find a valid record after %X/%X",
- LSN_FORMAT_ARGS(private.startptr));
+ LSN_FORMAT_ARGS(startptr));
/*
* Display a message that we're skipping data if `from` wasn't a pointer
* to the start of a record and also wasn't a pointer to the beginning of
* a segment (e.g. we were used in file mode).
*/
- if (first_record != private.startptr &&
- XLogSegmentOffset(private.startptr, WalSegSz) != 0)
+ if (first_record != startptr &&
+ XLogSegmentOffset(startptr, WalSegSz) != 0)
printf(ngettext("first record is after %X/%X, at %X/%X, skipping over %u byte\n",
"first record is after %X/%X, at %X/%X, skipping over %u bytes\n",
- (first_record - private.startptr)),
- LSN_FORMAT_ARGS(private.startptr),
+ (first_record - startptr)),
+ LSN_FORMAT_ARGS(startptr),
LSN_FORMAT_ARGS(first_record),
- (uint32) (first_record - private.startptr));
+ (uint32) (first_record - startptr));
for (;;)
{
@@ -1083,13 +1088,13 @@ main(int argc, char **argv)
while (XLogReadRecord(xlogreader_state, &record, &errormsg) ==
XLREAD_NEED_DATA)
{
- if (!WALDumpReadPage(xlogreader_state, (void *) &private))
+ if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr))
break;
}
if (!record)
{
- if (!config.follow || private.endptr_reached)
+ if (!config.follow)
break;
else
{
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1492f1992d..d8cb488820 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -56,6 +56,7 @@ typedef struct WALSegmentContext
} WALSegmentContext;
typedef struct XLogReaderState XLogReaderState;
+typedef struct XLogFindNextRecordState XLogFindNextRecordState;
/* Function type definition for the segment cleanup callback */
typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader);
@@ -240,6 +241,13 @@ struct XLogReaderState
char *errormsg_buf;
};
+struct XLogFindNextRecordState
+{
+ XLogReaderState *reader_state;
+ XLogRecPtr targetRecPtr;
+ XLogRecPtr currRecPtr;
+};
+
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
@@ -254,8 +262,8 @@ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
/* Function type definition for the read_page callback */
typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader,
void *private);
-extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
- XLogFindNextRecordCB read_page, void *private);
+extern XLogFindNextRecordState *InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr);
+extern bool XLogFindNextRecord(XLogFindNextRecordState *state);
#endif /* FRONTEND */
/* Read the next XLog record. Returns NULL on end-of-WAL or failure */
--
2.27.0
From 03c3fb95f3ce2985df40d955b155f844246b2557 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Wed, 7 Apr 2021 16:38:21 +0900
Subject: [PATCH v19 5/5] Split readLen and reqLen of XLogReaderState.
The variable is used as both out and in parameter of page-read
functions. Separate the varialbe according to the roles. To avoid
confusion between the two variables, provide a setter function for
page-read functions to set readLen.
---
src/backend/access/transam/xlog.c | 10 +++++-----
src/backend/access/transam/xlogreader.c | 17 ++++++++---------
src/backend/access/transam/xlogutils.c | 6 +++---
src/backend/replication/walsender.c | 6 +++---
src/bin/pg_rewind/parsexlog.c | 12 +++++++-----
src/bin/pg_waldump/pg_waldump.c | 6 +++---
src/include/access/xlogreader.h | 23 ++++++++++++++++-------
7 files changed, 45 insertions(+), 35 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9a9835f05d..d3d6fb4643 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -12101,7 +12101,7 @@ XLogPageRead(XLogReaderState *state,
{
char *readBuf = state->readBuf;
XLogRecPtr targetPagePtr = state->readPagePtr;
- int reqLen = state->readLen;
+ int reqLen = state->reqLen;
int readLen = 0;
XLogRecPtr targetRecPtr = state->ReadRecPtr;
uint32 targetPageOff;
@@ -12153,7 +12153,7 @@ retry:
close(readFile);
readFile = -1;
readSource = XLOG_FROM_ANY;
- state->readLen = -1;
+ XLogReaderNotifySize(state, -1);
return false;
}
}
@@ -12208,7 +12208,7 @@ retry:
pgstat_report_wait_end();
Assert(targetSegNo == state->seg.ws_segno);
- Assert(reqLen <= readLen);
+ Assert(readLen >= reqLen);
state->seg.ws_tli = curFileTLI;
@@ -12246,7 +12246,7 @@ retry:
}
Assert(state->readPagePtr == targetPagePtr);
- state->readLen = readLen;
+ XLogReaderNotifySize(state, readLen);
return true;
next_record_is_invalid:
@@ -12261,7 +12261,7 @@ next_record_is_invalid:
if (StandbyMode)
goto retry;
- state->readLen = -1;
+ XLogReaderNotifySize(state, -1);
return false;
}
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 89c59843b9..7d1b5b50e6 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -107,7 +107,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size,
waldir);
- /* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */
+ /* ReadRecPtr, EndRecPtr, reqLen and readLen initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -261,12 +261,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
* record being stored in *record. Otherwise *record is NULL.
*
* Returns XLREAD_NEED_DATA if more data is needed to finish reading the
- * current record. In that case, state->readPagePtr and state->readLen inform
+ * current record. In that case, state->readPagePtr and state->reqLen inform
* the desired position and minimum length of data needed. The caller shall
* read in the requested data and set state->readBuf to point to a buffer
* containing it. The caller must also set state->seg->ws_tli and
* state->readLen to indicate the timeline that it was read from, and the
- * length of data that is now available (which must be >= given readLen),
+ * length of data that is now available (which must be >= given reqLen),
* respectively.
*
* If invalid data is encountered, returns XLREAD_FAIL with *record being set to
@@ -630,7 +630,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* XLogNeedData should have ensured that the whole page
* header was read
*/
- Assert(state->readLen >= pageHeaderSize);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
record_len = XLOG_BLCKSZ - pageHeaderSize;
@@ -643,7 +643,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
* XLogNeedData should have ensured all needed data was
* read
*/
- Assert(state->readLen >= request_len);
+ Assert(request_len <= state->readLen);
memcpy(state->readRecordBuf + state->recordGotLen,
(char *) contdata, record_len);
@@ -696,7 +696,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize);
}
- Assert(!*record || state->readLen >= 0);
if (DecodeXLogRecord(state, *record, errormsg))
return XLREAD_SUCCESS;
@@ -763,7 +762,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
/* Request more data if we don't have the full header. */
if (state->readLen < pageHeaderSize)
{
- state->readLen = pageHeaderSize;
+ state->reqLen = pageHeaderSize;
return true;
}
@@ -840,7 +839,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* will not come back here, but will request the actual target page.
*/
state->readPagePtr = pageptr - targetPageOff;
- state->readLen = XLOG_BLCKSZ;
+ state->reqLen = XLOG_BLCKSZ;
return true;
}
@@ -849,7 +848,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
* header so that we can validate it.
*/
state->readPagePtr = pageptr;
- state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ state->reqLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
return true;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b003990745..61361192e7 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -828,7 +828,7 @@ bool
read_local_xlog_page(XLogReaderState *state)
{
XLogRecPtr targetPagePtr = state->readPagePtr;
- int reqLen = state->readLen;
+ int reqLen = state->reqLen;
char *cur_page = state->readBuf;
XLogRecPtr read_upto,
loc;
@@ -928,7 +928,7 @@ read_local_xlog_page(XLogReaderState *state)
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- state->readLen = -1;
+ XLogReaderNotifySize(state, -1);
return false;
}
else
@@ -948,7 +948,7 @@ read_local_xlog_page(XLogReaderState *state)
/* number of valid bytes in the buffer */
state->readPagePtr = targetPagePtr;
- state->readLen = count;
+ XLogReaderNotifySize(state, count);
return true;
}
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index b024bbc3cd..31522c8cc2 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -807,7 +807,7 @@ static bool
logical_read_xlog_page(XLogReaderState *state)
{
XLogRecPtr targetPagePtr = state->readPagePtr;
- int reqLen = state->readLen;
+ int reqLen = state->reqLen;
char *cur_page = state->readBuf;
XLogRecPtr flushptr;
int count;
@@ -826,7 +826,7 @@ logical_read_xlog_page(XLogReaderState *state)
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
{
- state->readLen = -1;
+ XLogReaderNotifySize(state, -1);
return false;
}
@@ -856,7 +856,7 @@ logical_read_xlog_page(XLogReaderState *state)
XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize);
CheckXLogRemoved(segno, state->seg.ws_tli);
- state->readLen = count;
+ XLogReaderNotifySize(state, count);
return true;
}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..707493dddf 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -254,6 +254,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
XLogSegNo targetSegNo;
int r;
+ Assert(xlogreader->reqLen <= XLOG_BLCKSZ);
+
XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
@@ -304,7 +306,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
if (restoreCommand == NULL)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- xlogreader->readLen = -1;
+ XLogReaderNotifySize(xlogreader, -1);
return false;
}
@@ -319,7 +321,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
if (xlogreadfd < 0)
{
- xlogreader->readLen = -1;
+ XLogReaderNotifySize(xlogreader, -1);
return false;
}
else
@@ -337,7 +339,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- xlogreader->readLen = -1;
+ XLogReaderNotifySize(xlogreader, -1);
return false;
}
@@ -351,14 +353,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- xlogreader->readLen = -1;
+ XLogReaderNotifySize(xlogreader, -1);
return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli;
- xlogreader->readLen = XLOG_BLCKSZ;
+ XLogReaderNotifySize(xlogreader, XLOG_BLCKSZ);
return true;
}
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ab2d079bdb..82af398d20 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -332,7 +332,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline,
XLogRecPtr startptr, XLogRecPtr endptr)
{
XLogRecPtr targetPagePtr = state->readPagePtr;
- int reqLen = state->readLen;
+ int reqLen = state->reqLen;
char *readBuff = state->readBuf;
int count = XLOG_BLCKSZ;
WALReadError errinfo;
@@ -347,7 +347,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline,
else
{
/* Notify xlogreader that we didn't read at all */
- state->readLen = -1;
+ XLogReaderNotifySize(state, -1);
return false;
}
}
@@ -377,7 +377,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline,
}
/* Notify xlogreader of how many bytes we have read */
- state->readLen = count;
+ XLogReaderNotifySize(state, count);
return true;
}
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index d8cb488820..836ca7fce8 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -149,19 +149,21 @@ struct XLogReaderState
/* ----------------------------------------
* Communication with page reader
- * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least reqLen bytes.
* ----------------------------------------
*/
- /* variables to communicate with page reader */
+ /* variables to inform to the callers from xlogreader */
XLogRecPtr readPagePtr; /* page pointer to read */
- int32 readLen; /* bytes requested to reader, or actual bytes
- * read by reader, which must be larger than
- * the request, or -1 on error */
+ int32 reqLen; /* bytes requested to the caller */
char *readBuf; /* buffer to store data */
bool page_verified; /* is the page header on the buffer verified? */
- bool record_verified; /* is the current record header verified? */
-
+ bool record_verified;/* is the current record header verified? */
+ /* variables to respond from the callers to xlogreader */
+ int32 readLen; /* actual bytes read by reader, which must be
+ * larger than the request, or -1 on error.
+ * Use XLogReaderNotifyLength() to set a
+ * value. */
/* ----------------------------------------
* Decoded representation of current record
@@ -248,6 +250,13 @@ struct XLogFindNextRecordState
XLogRecPtr currRecPtr;
};
+/* setter functions of XLogReaderState used by other modules */
+static inline void
+XLogReaderNotifySize(XLogReaderState *state, int32 len)
+{
+ state->readLen = len;
+}
+
/* Get a new XLogReader */
extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
const char *waldir,
--
2.27.0
On Wed, Apr 7, 2021 at 8:50 PM Kyotaro Horiguchi
<horikyota.ntt@gmail.com> wrote:
> I haven't changed the name "XLog reader" to "XLog decoder". I'm doing
> that but it affects somewhat wide range of code.
Thanks for the new patch set! Let's not worry about renaming it for now.
This fails in check-world as seen on cfbot; I am not 100% sure but
this change fixes it:
@@ -1231,7 +1231,7 @@ XLogFindNextRecord(XLogFindNextRecordState *state)
{
/* Rewind the reader to the beginning of the
last record. */
state->currRecPtr = state->reader_state->ReadRecPtr;
- XLogBeginRead(state->reader_state, found);
+ XLogBeginRead(state->reader_state, state->currRecPtr);
The variable "found" seem to be useless.
I still see the 3 warnings mentioned earlier when compiling without
--enable-cassert.
There is a stray elog(HOGE) :-)
I squashed the patch set into one because half of them were fixups, and the two main patches were really parts of the same change and should go in together. I fixed a few compiler warnings (GCC 10.2 reported several uninitialised variables, comparisons that are always true, etc) and some comments. You can see these in the fixup patch. +static inline void +XLogReaderNotifySize(XLogReaderState *state, int32 len) I think maybe it it should really be XLogReaderSetInputData(state, tli, data, size) in a later release. In the meantime, I changed it to XLogReaderSetInputData(state, size), hope that name make sense...
Вложения
On Thu, Apr 8, 2021 at 9:46 PM Thomas Munro <thomas.munro@gmail.com> wrote: > I squashed the patch set into one because half of them were fixups, > and the two main patches were really parts of the same change and > should go in together. > > I fixed a few compiler warnings (GCC 10.2 reported several > uninitialised variables, comparisons that are always true, etc) and > some comments. You can see these in the fixup patch. Pushed. Luckily there are plenty more improvements possible for XLogReader/XLogDecoder in the next cycle.
At Thu, 8 Apr 2021 23:51:34 +1200, Thomas Munro <thomas.munro@gmail.com> wrote in > On Thu, Apr 8, 2021 at 9:46 PM Thomas Munro <thomas.munro@gmail.com> wrote: > > I squashed the patch set into one because half of them were fixups, > > and the two main patches were really parts of the same change and > > should go in together. > > > > I fixed a few compiler warnings (GCC 10.2 reported several > > uninitialised variables, comparisons that are always true, etc) and > > some comments. You can see these in the fixup patch. > > Pushed. Luckily there are plenty more improvements possible for > XLogReader/XLogDecoder in the next cycle. I'm surprised to see this pushed this soon. Thanks for pushing this! And thanks for fixing the remaining mistakes including some stupid ones.. At Thu, 8 Apr 2021 10:04:26 +1200, Thomas Munro <thomas.munro@gmail.com> wrote in > There is a stray elog(HOGE) :-) Ugggghhhh! This looks like getting slipped-in while investigating another issue.. Thanks for preventing the repository from being contaminated by such a thing.. At Thu, 8 Apr 2021 21:46:06 +1200, Thomas Munro <thomas.munro@gmail.com> wrote in > I think maybe it it should really be XLogReaderSetInputData(state, > tli, data, size) in a later release. In the meantime, I changed it to > XLogReaderSetInputData(state, size), hope that name make sense... Sounds better. I didn't like that page-readers are allowed to touch XLogReaderStats.seg directly. Anyway it would be a small change. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
At Fri, 09 Apr 2021 09:36:59 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > I'm surprised to see this pushed this soon. Thanks for pushing this! Then this has been reverted. I'm not sure how to check for the possible defect happens on that platform, but, anyways I reverted the CF item to "Needs Review" then moved to the next CF. Maybe I will rebase it soon. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
On Wed, Jun 30, 2021 at 12:54 PM Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote:
At Fri, 09 Apr 2021 09:36:59 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in
> I'm surprised to see this pushed this soon. Thanks for pushing this!
Then this has been reverted. I'm not sure how to check for the
possible defect happens on that platform, but, anyways I reverted the
CF item to "Needs Review" then moved to the next CF.
Maybe I will rebase it soon.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
At Thu, 15 Jul 2021 00:39:52 +0500, Ibrar Ahmed <ibrar.ahmad@gmail.com> wrote in > On Wed, Jun 30, 2021 at 12:54 PM Kyotaro Horiguchi <horikyota.ntt@gmail.com> > wrote: > > Maybe I will rebase it soon. > > > > Yes, rebase is required, therefore I am changing the status to "Waiting On > Author" > http://cfbot.cputube.org/patch_33_2113.log Gah... Thank you for noticing me. I thought that I have sent the rebased version. This is the rebased version on the current master. regards. -- Kyotaro Horiguchi NTT Open Source Software Center From cc36aefa3943e67d357be989044c951b5e6c0702 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Wed, 7 Jul 2021 13:10:34 +0900 Subject: [PATCH v21] Remove read_page callback from XLogReadRecord. Previously, the XLogReader module would fetch new input data using a callback function. Redesign the interface so that it tells the caller to insert more data with a return value instead. This API suits later patches for prefetching, encryption and probably other future features that would otherwise require extending the callback interface for new projects. As incidental cleanup work, move global variables readOff, readLen and readSegNo inside XlogReaderState. Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> Author: Heikki Linnakangas <hlinnaka@iki.fi> (earlier version) Reviewed-by: Antonin Houska <ah@cybertec.at> Reviewed-by: Alvaro Herrera <alvherre@2ndquadrant.com> Reviewed-by: Takashi Menjo <takashi.menjo@gmail.com> Reviewed-by: Andres Freund <andres@anarazel.de> Reviewed-by: Thomas Munro <thomas.munro@gmail.com> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/twophase.c | 14 +- src/backend/access/transam/xlog.c | 125 +-- src/backend/access/transam/xlogreader.c | 968 +++++++++++------- src/backend/access/transam/xlogutils.c | 25 +- src/backend/replication/logical/logical.c | 26 +- .../replication/logical/logicalfuncs.c | 13 +- src/backend/replication/slotfuncs.c | 18 +- src/backend/replication/walsender.c | 42 +- src/bin/pg_rewind/parsexlog.c | 105 +- src/bin/pg_waldump/pg_waldump.c | 141 +-- src/include/access/xlogreader.h | 157 +-- src/include/access/xlogutils.h | 4 +- src/include/pg_config_manual.h | 2 +- src/include/replication/logical.h | 11 +- 14 files changed, 955 insertions(+), 696 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 6d3efb49a4..e0c1e8f334 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) char *errormsg; TimeLineID save_currtli = ThisTimeLineID; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.page_read = &read_local_xlog_page, - .segment_open = &wal_segment_open, - .segment_close = &wal_segment_close), - NULL); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -1342,7 +1339,12 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errdetail("Failed while allocating a WAL reading processor."))); XLogBeginRead(xlogreader, lsn); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!read_local_xlog_page(xlogreader)) + break; + } /* * Restore immediately the timeline where it was previously, as diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c7c928f50b..f9aea2a689 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -819,17 +819,13 @@ static XLogSegNo openLogSegNo = 0; /* * These variables are used similarly to the ones above, but for reading - * the XLOG. readOff is the offset of the page just read, readLen - * indicates how much of it has been read into readBuf, and readSource + * the XLOG. readOff is the offset of the page just read, and readSource * indicates where we got the currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -846,13 +842,6 @@ static XLogSource currentSource = XLOG_FROM_ANY; static bool lastSourceFailed = false; static bool pendingWalRcvRestart = false; -typedef struct XLogPageReadPrivate -{ - int emode; - bool fetching_ckpt; /* are we fetching a checkpoint record? */ - bool randAccess; -} XLogPageReadPrivate; - /* * These variables track when we last obtained some WAL data to process, * and where we got it from. (XLogReceiptSource is initially the same as @@ -927,10 +916,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf); +static bool XLogPageRead(XLogReaderState *state, + bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static void XLogShutdownWalRcv(void); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); @@ -1242,8 +1233,7 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(), NULL); + debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!debug_reader) { @@ -4394,16 +4384,10 @@ CleanupBackupHistory(void) * record is available. */ static XLogRecord * -ReadRecord(XLogReaderState *xlogreader, int emode, - bool fetching_ckpt) +ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt) { XLogRecord *record; - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - - /* Pass through parameters to XLogPageRead */ - private->fetching_ckpt = fetching_ckpt; - private->emode = emode; - private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr); + bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr); /* This is the first attempt to read this page. */ lastSourceFailed = false; @@ -4411,8 +4395,16 @@ ReadRecord(XLogReaderState *xlogreader, int emode, for (;;) { char *errormsg; + XLogReadRecordResult result; + + while ((result = XLogReadRecord(xlogreader, &record, &errormsg)) + == XLREAD_NEED_DATA) + { + if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess)) + break; + + } - record = XLogReadRecord(xlogreader, &errormsg); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) @@ -6489,7 +6481,6 @@ StartupXLOG(void) bool backupFromStandby = false; DBState dbstate_at_startup; XLogReaderState *xlogreader; - XLogPageReadPrivate private; bool promoted = false; struct stat st; @@ -6648,13 +6639,9 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - MemSet(&private, 0, sizeof(XLogPageReadPrivate)); xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.page_read = &XLogPageRead, - .segment_open = NULL, - .segment_close = wal_segment_close), - &private); + XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -7857,7 +7844,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -12147,13 +12135,15 @@ CancelBackup(void) * XLogPageRead() to try fetching the record from another source, or to * sleep and retry. */ -static int -XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf) +static bool +XLogPageRead(XLogReaderState *state, + bool fetching_ckpt, int emode, bool randAccess) { - XLogPageReadPrivate *private = - (XLogPageReadPrivate *) xlogreader->private_data; - int emode = private->emode; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->reqLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -12166,7 +12156,7 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12174,10 +12164,10 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12187,7 +12177,7 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12196,17 +12186,15 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - private->randAccess, - private->fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->seg.ws_segno)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - return -1; + XLogReaderNotifySize(state, -1); + return false; } } @@ -12233,40 +12221,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->seg.ws_segno, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); - Assert(reqLen <= readLen); + Assert(targetSegNo == state->seg.ws_segno); + Assert(readLen >= reqLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12295,14 +12279,16 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - return readLen; + Assert(state->readPagePtr == targetPagePtr); + XLogReaderNotifySize(state, readLen); + return true; next_record_is_invalid: lastSourceFailed = true; @@ -12310,14 +12296,14 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - else - return -1; + + XLogReaderNotifySize(state, -1); + return false; } /* @@ -12348,7 +12334,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 5cf74e181a..20cbd797a4 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -39,11 +39,11 @@ static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); -static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, - int reqLen); +static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, + int reqLen, bool header_inclusive); static void XLogReaderInvalReadState(XLogReaderState *state); static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, - XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); + XLogRecPtr PrevRecPtr, XLogRecord *record); static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr); static void ResetDecoder(XLogReaderState *state); @@ -76,7 +76,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) */ XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, - XLogReaderRoutine *routine, void *private_data) + WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,7 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, return NULL; /* initialize caller-provided support functions */ - state->routine = *routine; + state->cleanup_cb = cleanup_cb; state->max_block_id = -1; @@ -110,9 +110,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size, waldir); - /* system_identifier initialized to zeroes above */ - state->private_data = private_data; - /* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */ + /* ReadRecPtr, EndRecPtr, reqLen and readLen initialized to zeroes above */ state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1, MCXT_ALLOC_NO_OOM); if (!state->errormsg_buf) @@ -143,8 +141,8 @@ XLogReaderFree(XLogReaderState *state) { int block_id; - if (state->seg.ws_file != -1) - state->routine.segment_close(state); + if (state->seg.ws_file >= 0) + state->cleanup_cb(state); for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++) { @@ -249,6 +247,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) /* Begin at the passed-in record pointer. */ state->EndRecPtr = RecPtr; state->ReadRecPtr = InvalidXLogRecPtr; + state->readRecordState = XLREAD_NEXT_RECORD; } /* @@ -257,303 +256,459 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) * XLogBeginRead() or XLogFindNextRecord() must be called before the first call * to XLogReadRecord(). * - * If the page_read callback fails to read the requested data, NULL is - * returned. The callback is expected to have reported the error; errormsg - * is set to NULL. + * This function may return XLREAD_NEED_DATA several times before returning a + * result record. The caller shall read in some new data then call this + * function again with the same parameters. * - * If the reading fails for some other reason, NULL is also returned, and - * *errormsg is set to a string with details of the failure. + * When a record is successfully read, returns XLREAD_SUCCESS with result + * record being stored in *record. Otherwise *record is NULL. * + * Returns XLREAD_NEED_DATA if more data is needed to finish reading the + * current record. In that case, state->readPagePtr and state->reqLen inform + * the desired position and minimum length of data needed. The caller shall + * read in the requested data and set state->readBuf to point to a buffer + * containing it. The caller must also set state->seg->ws_tli and + * state->readLen to indicate the timeline that it was read from, and the + * length of data that is now available (which must be >= given reqLen), + * respectively. + * + * If invalid data is encountered, returns XLREAD_FAIL with *record being set to + * NULL. *errormsg is set to a string with details of the failure. * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. + * + * + * This function runs a state machine consists of the following states. + * + * XLREAD_NEXT_RECORD : + * The initial state, if called with valid RecPtr, try to read a record at + * that position. If invalid RecPtr is given try to read a record just after + * the last one previously read. + * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN. + * + * XLREAD_TOT_LEN: + * Examining record header. Ends after reading record total + * length. recordRemainLen and recordGotLen are initialized. + * + * XLREAD_FIRST_FRAGMENT: + * Reading the first fragment. Ends with finishing reading a single + * record. Goes to XLREAD_NEXT_RECORD if that's all or + * XLREAD_CONTINUATION if we have continuation. + + * XLREAD_CONTINUATION: + * Reading continuation of record. Ends with finishing the whole record then + * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates + * how much is left and readRecordBuf holds the partially assert + * record.recordContRecPtr points to the beginning of the next page where to + * continue. + * + * If wrong data found in any state, the state machine stays at the current + * state. This behavior allows to continue reading a reacord switching among + * different souces, while streaming replication. */ -XLogRecord * -XLogReadRecord(XLogReaderState *state, char **errormsg) +XLogReadRecordResult +XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) { - XLogRecPtr RecPtr; - XLogRecord *record; - XLogRecPtr targetPagePtr; - bool randAccess; - uint32 len, - total_len; - uint32 targetRecOff; - uint32 pageHeaderSize; - bool gotheader; - int readOff; + XLogRecord *prec; - /* - * randAccess indicates whether to verify the previous-record pointer of - * the record we're reading. We only do this if we're reading - * sequentially, which is what we initially assume. - */ - randAccess = false; + *record = NULL; /* reset error state */ *errormsg = NULL; state->errormsg_buf[0] = '\0'; - ResetDecoder(state); - - RecPtr = state->EndRecPtr; - - if (state->ReadRecPtr != InvalidXLogRecPtr) - { - /* read the record after the one we just read */ - - /* - * EndRecPtr is pointing to end+1 of the previous WAL record. If - * we're at a page boundary, no more records can fit on the current - * page. We must skip over the page header, but we can't do that until - * we've read in the page, since the header size is variable. - */ - } - else - { - /* - * Caller supplied a position to start at. - * - * In this case, EndRecPtr should already be pointing to a valid - * record starting position. - */ - Assert(XRecOffIsValid(RecPtr)); - randAccess = true; - } - - state->currRecPtr = RecPtr; - - targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ); - targetRecOff = RecPtr % XLOG_BLCKSZ; - - /* - * Read the page containing the record into state->readBuf. Request enough - * byte to cover the whole record header, or at least the part of it that - * fits on the same page. - */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); - if (readOff < 0) - goto err; - - /* - * ReadPageInternal always returns at least the page header, so we can - * examine it now. - */ - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); - if (targetRecOff == 0) + switch (state->readRecordState) { - /* - * At page start, so skip over page header. - */ - RecPtr += pageHeaderSize; - targetRecOff = pageHeaderSize; - } - else if (targetRecOff < pageHeaderSize) - { - report_invalid_record(state, "invalid record offset at %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; - } + case XLREAD_NEXT_RECORD: + ResetDecoder(state); - if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) && - targetRecOff == pageHeaderSize) - { - report_invalid_record(state, "contrecord is requested by %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; - } - - /* ReadPageInternal has verified the page header */ - Assert(pageHeaderSize <= readOff); - - /* - * Read the record length. - * - * NB: Even though we use an XLogRecord pointer here, the whole record - * header might not fit on this page. xl_tot_len is the first field of the - * struct, so it must be on this page (the records are MAXALIGNed), but we - * cannot access any other fields until we've verified that we got the - * whole header. - */ - record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); - total_len = record->xl_tot_len; - - /* - * If the whole record header is on this page, validate it immediately. - * Otherwise do just a basic sanity check on xl_tot_len, and validate the - * rest of the header after reading it from the next page. The xl_tot_len - * check is necessary here to ensure that we enter the "Need to reassemble - * record" code path below; otherwise we might fail to apply - * ValidXLogRecordHeader at all. - */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) - { - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record, - randAccess)) - goto err; - gotheader = true; - } - else - { - /* XXX: more validation should be done here */ - if (total_len < SizeOfXLogRecord) - { - report_invalid_record(state, - "invalid record length at %X/%X: wanted %u, got %u", - LSN_FORMAT_ARGS(RecPtr), - (uint32) SizeOfXLogRecord, total_len); - goto err; - } - gotheader = false; - } - - len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ; - if (total_len > len) - { - /* Need to reassemble record */ - char *contdata; - XLogPageHeader pageHeader; - char *buffer; - uint32 gotlen; - - /* - * Enlarge readRecordBuf as needed. - */ - if (total_len > state->readRecordBufSize && - !allocate_recordbuf(state, total_len)) - { - /* We treat this as a "bogus data" condition */ - report_invalid_record(state, "record length %u at %X/%X too long", - total_len, LSN_FORMAT_ARGS(RecPtr)); - goto err; - } - - /* Copy the first fragment of the record from the first page. */ - memcpy(state->readRecordBuf, - state->readBuf + RecPtr % XLOG_BLCKSZ, len); - buffer = state->readRecordBuf + len; - gotlen = len; - - do - { - /* Calculate pointer to beginning of next page */ - targetPagePtr += XLOG_BLCKSZ; - - /* Wait for the next page to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(total_len - gotlen + SizeOfXLogShortPHD, - XLOG_BLCKSZ)); - - if (readOff < 0) - goto err; - - Assert(SizeOfXLogShortPHD <= readOff); - - /* Check that the continuation on next page looks valid */ - pageHeader = (XLogPageHeader) state->readBuf; - if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) + if (state->ReadRecPtr != InvalidXLogRecPtr) + { + /* read the record after the one we just read */ + + /* + * EndRecPtr is pointing to end+1 of the previous WAL record. + * If we're at a page boundary, no more records can fit on the + * current page. We must skip over the page header, but we + * can't do that until we've read in the page, since the + * header size is variable. + */ + state->PrevRecPtr = state->ReadRecPtr; + state->ReadRecPtr = state->EndRecPtr; + } + else { - report_invalid_record(state, - "there is no contrecord flag at %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; + /* + * Caller supplied a position to start at. + * + * In this case, EndRecPtr should already be pointing to a + * valid record starting position. + */ + Assert(XRecOffIsValid(state->EndRecPtr)); + state->ReadRecPtr = state->EndRecPtr; + + /* + * We cannot verify the previous-record pointer when we're + * seeking to a particular record. Reset PrevRecPtr so that we + * won't try doing that. + */ + state->PrevRecPtr = InvalidXLogRecPtr; + state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */ } - /* - * Cross-check that xlp_rem_len agrees with how much of the record - * we expect there to be left. - */ - if (pageHeader->xlp_rem_len == 0 || - total_len != (pageHeader->xlp_rem_len + gotlen)) + state->record_verified = false; + state->readRecordState = XLREAD_TOT_LEN; + /* fall through */ + + case XLREAD_TOT_LEN: { - report_invalid_record(state, - "invalid contrecord length %u (expected %lld) at %X/%X", - pageHeader->xlp_rem_len, - ((long long) total_len) - gotlen, - LSN_FORMAT_ARGS(RecPtr)); - goto err; + uint32 total_len; + uint32 pageHeaderSize; + XLogRecPtr targetPagePtr; + uint32 targetRecOff; + XLogPageHeader pageHeader; + + targetPagePtr = + state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); + targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + + /* + * Check if we have enough data. For the first record in the + * page, the requesting length doesn't contain page header. + */ + if (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + targetRecOff != 0)) + return XLREAD_NEED_DATA; + + /* error out if caller supplied bogus page */ + if (!state->page_verified) + goto err; + + /* examine page header now. */ + pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + if (targetRecOff == 0) + { + /* At page start, so skip over page header. */ + state->ReadRecPtr += pageHeaderSize; + targetRecOff = pageHeaderSize; + } + else if (targetRecOff < pageHeaderSize) + { + report_invalid_record(state, "invalid record offset at %X/%X", + LSN_FORMAT_ARGS(state->ReadRecPtr)); + goto err; + } + + pageHeader = (XLogPageHeader) state->readBuf; + if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) && + targetRecOff == pageHeaderSize) + { + report_invalid_record(state, "contrecord is requested by %X/%X", + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr); + goto err; + } + + /* XLogNeedData has verified the page header */ + Assert(pageHeaderSize <= state->readLen); + + /* + * Read the record length. + * + * NB: Even though we use an XLogRecord pointer here, the + * whole record header might not fit on this page. xl_tot_len + * is the first field of the struct, so it must be on this + * page (the records are MAXALIGNed), but we cannot access any + * other fields until we've verified that we got the whole + * header. + */ + prec = (XLogRecord *) (state->readBuf + + state->ReadRecPtr % XLOG_BLCKSZ); + total_len = prec->xl_tot_len; + + /* + * If the whole record header is on this page, validate it + * immediately. Otherwise do just a basic sanity check on + * xl_tot_len, and validate the rest of the header after + * reading it from the next page. The xl_tot_len check is + * necessary here to ensure that we enter the + * XLREAD_CONTINUATION state below; otherwise we might fail to + * apply ValidXLogRecordHeader at all. + */ + if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + { + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, prec)) + goto err; + + state->record_verified = true; + } + else + { + /* XXX: more validation should be done here */ + if (total_len < SizeOfXLogRecord) + { + report_invalid_record(state, + "invalid record length at %X/%X: wanted %u, got %u", + LSN_FORMAT_ARGS(state->ReadRecPtr), + (uint32) SizeOfXLogRecord, total_len); + goto err; + } + } + + /* + * Wait for the rest of the record, or the part of the record + * that fit on the first page if crossed a page boundary, to + * become available. + */ + state->recordGotLen = 0; + state->recordRemainLen = total_len; + state->readRecordState = XLREAD_FIRST_FRAGMENT; } + /* fall through */ - /* Append the continuation from this page to the buffer */ - pageHeaderSize = XLogPageHeaderSize(pageHeader); + case XLREAD_FIRST_FRAGMENT: + { + uint32 total_len = state->recordRemainLen; + uint32 request_len; + uint32 record_len; + XLogRecPtr targetPagePtr; + uint32 targetRecOff; + + /* + * Wait for the rest of the record on the first page to become + * available + */ + targetPagePtr = + state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); + targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + + request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + record_len = request_len - targetRecOff; + + /* ReadRecPtr contains page header */ + Assert(targetRecOff != 0); + if (XLogNeedData(state, targetPagePtr, request_len, true)) + return XLREAD_NEED_DATA; + + /* error out if caller supplied bogus page */ + if (!state->page_verified) + goto err; - if (readOff < pageHeaderSize) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize); + prec = (XLogRecord *) (state->readBuf + targetRecOff); - Assert(pageHeaderSize <= readOff); + /* validate record header if not yet */ + if (!state->record_verified && record_len >= SizeOfXLogRecord) + { + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, prec)) + goto err; - contdata = (char *) state->readBuf + pageHeaderSize; - len = XLOG_BLCKSZ - pageHeaderSize; - if (pageHeader->xlp_rem_len < len) - len = pageHeader->xlp_rem_len; + state->record_verified = true; + } - if (readOff < pageHeaderSize + len) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize + len); - memcpy(buffer, (char *) contdata, len); - buffer += len; - gotlen += len; + if (total_len == record_len) + { + /* Record does not cross a page boundary */ + Assert(state->record_verified); - /* If we just reassembled the record header, validate it. */ - if (!gotheader) - { - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, - record, randAccess)) + if (!ValidXLogRecord(state, prec, state->ReadRecPtr)) + goto err; + + state->record_verified = true; /* to be tidy */ + + /* We already checked the header earlier */ + state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len); + + *record = prec; + state->readRecordState = XLREAD_NEXT_RECORD; + break; + } + + /* + * The record continues on the next page. Need to reassemble + * record + */ + Assert(total_len > record_len); + + /* Enlarge readRecordBuf as needed. */ + if (total_len > state->readRecordBufSize && + !allocate_recordbuf(state, total_len)) + { + /* We treat this as a "bogus data" condition */ + report_invalid_record(state, + "record length %u at %X/%X too long", + total_len, + LSN_FORMAT_ARGS(state->ReadRecPtr)); goto err; - gotheader = true; - } - } while (gotlen < total_len); + } - Assert(gotheader); + /* Copy the first fragment of the record from the first page. */ + memcpy(state->readRecordBuf, state->readBuf + targetRecOff, + record_len); + state->recordGotLen += record_len; + state->recordRemainLen -= record_len; - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecord(state, record, RecPtr)) - goto err; + /* Calculate pointer to beginning of next page */ + state->recordContRecPtr = state->ReadRecPtr + record_len; + Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); - state->ReadRecPtr = RecPtr; - state->EndRecPtr = targetPagePtr + pageHeaderSize - + MAXALIGN(pageHeader->xlp_rem_len); - } - else - { - /* Wait for the record data to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + total_len, XLOG_BLCKSZ)); - if (readOff < 0) - goto err; + state->readRecordState = XLREAD_CONTINUATION; + } + /* fall through */ - /* Record does not cross a page boundary */ - if (!ValidXLogRecord(state, record, RecPtr)) - goto err; + case XLREAD_CONTINUATION: + { + XLogPageHeader pageHeader; + uint32 pageHeaderSize; + XLogRecPtr targetPagePtr; + + /* + * we enter this state only if we haven't read the whole + * record. + */ + Assert(state->recordRemainLen > 0); + + while (state->recordRemainLen > 0) + { + char *contdata; + uint32 request_len; + uint32 record_len; + + /* Wait for the next page to become available */ + targetPagePtr = state->recordContRecPtr; + + /* this request contains page header */ + Assert(targetPagePtr != 0); + if (XLogNeedData(state, targetPagePtr, + Min(state->recordRemainLen, XLOG_BLCKSZ), + false)) + return XLREAD_NEED_DATA; + + if (!state->page_verified) + goto err; + + Assert(SizeOfXLogShortPHD <= state->readLen); + + /* Check that the continuation on next page looks valid */ + pageHeader = (XLogPageHeader) state->readBuf; + if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) + { + report_invalid_record( + state, + "there is no contrecord flag at %X/%X reading %X/%X", + (uint32) (state->recordContRecPtr >> 32), + (uint32) state->recordContRecPtr, + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr); + goto err; + } + + /* + * Cross-check that xlp_rem_len agrees with how much of + * the record we expect there to be left. + */ + if (pageHeader->xlp_rem_len == 0 || + pageHeader->xlp_rem_len != state->recordRemainLen) + { + report_invalid_record( + state, + "invalid contrecord length %u at %X/%X reading %X/%X, expected %u", + pageHeader->xlp_rem_len, + (uint32) (state->recordContRecPtr >> 32), + (uint32) state->recordContRecPtr, + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr, + state->recordRemainLen); + goto err; + } + + /* Append the continuation from this page to the buffer */ + pageHeaderSize = XLogPageHeaderSize(pageHeader); + + /* + * XLogNeedData should have ensured that the whole page + * header was read + */ + Assert(pageHeaderSize <= state->readLen); + + contdata = (char *) state->readBuf + pageHeaderSize; + record_len = XLOG_BLCKSZ - pageHeaderSize; + if (pageHeader->xlp_rem_len < record_len) + record_len = pageHeader->xlp_rem_len; + + request_len = record_len + pageHeaderSize; + + /* + * XLogNeedData should have ensured all needed data was + * read + */ + Assert(request_len <= state->readLen); + + memcpy(state->readRecordBuf + state->recordGotLen, + (char *) contdata, record_len); + state->recordGotLen += record_len; + state->recordRemainLen -= record_len; + + /* If we just reassembled the record header, validate it. */ + if (!state->record_verified) + { + Assert(state->recordGotLen >= SizeOfXLogRecord); + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, + (XLogRecord *) state->readRecordBuf)) + goto err; + + state->record_verified = true; + } + + /* + * Calculate pointer to beginning of next page, and + * continue + */ + state->recordContRecPtr += XLOG_BLCKSZ; + } + + /* targetPagePtr is pointing the last-read page here */ + prec = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecord(state, prec, state->ReadRecPtr)) + goto err; - state->EndRecPtr = RecPtr + MAXALIGN(total_len); + pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + state->EndRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(pageHeader->xlp_rem_len); - state->ReadRecPtr = RecPtr; + *record = prec; + state->readRecordState = XLREAD_NEXT_RECORD; + break; + } } /* * Special processing if it's an XLOG SWITCH record */ - if (record->xl_rmid == RM_XLOG_ID && - (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) + if ((*record)->xl_rmid == RM_XLOG_ID && + ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) { /* Pretend it extends to end of segment */ state->EndRecPtr += state->segcxt.ws_segsize - 1; state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize); } - if (DecodeXLogRecord(state, record, errormsg)) - return record; - else - return NULL; + if (DecodeXLogRecord(state, *record, errormsg)) + return XLREAD_SUCCESS; + + *record = NULL; + return XLREAD_FAIL; err: /* - * Invalidate the read state. We might read from a different source after + * Invalidate the read page. We might read from a different source after * failure. */ XLogReaderInvalReadState(state); @@ -561,113 +716,143 @@ err: if (state->errormsg_buf[0] != '\0') *errormsg = state->errormsg_buf; - return NULL; + *record = NULL; + return XLREAD_FAIL; } /* - * Read a single xlog page including at least [pageptr, reqLen] of valid data - * via the page_read() callback. + * Checks that an xlog page loaded in state->readBuf is including at least + * [pageptr, reqLen] and the page is valid. header_inclusive indicates that + * reqLen is calculated including page header length. * - * Returns -1 if the required page cannot be read for some reason; errormsg_buf - * is set in that case (unless the error occurs in the page_read callback). + * Returns false if the buffer already contains the requested data, or found + * error. state->page_verified is set to true for the former and false for the + * latter. * - * We fetch the page from a reader-local cache if we know we have the required - * data and if there hasn't been any error since caching the data. + * Otherwise returns true and requests data loaded onto state->readBuf by + * state->readPagePtr and state->readLen. The caller shall call this function + * again after filling the buffer at least with that portion of data and set + * state->readLen to the length of actually loaded data. + * + * If header_inclusive is false, corrects reqLen internally by adding the + * actual page header length and may request caller for new data. */ -static int -ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) +static bool +XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, + bool header_inclusive) { - int readLen; uint32 targetPageOff; XLogSegNo targetSegNo; - XLogPageHeader hdr; + uint32 addLen = 0; - Assert((pageptr % XLOG_BLCKSZ) == 0); + /* Some data is loaded, but page header is not verified yet. */ + if (!state->page_verified && + !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0) + { + uint32 pageHeaderSize; + /* just loaded new data so needs to verify page header */ + + /* The caller must have loaded at least page header */ + Assert(state->readLen >= SizeOfXLogShortPHD); + + /* + * We have enough data to check the header length. Recheck the loaded + * length against the actual header length. + */ + pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); + + /* Request more data if we don't have the full header. */ + if (state->readLen < pageHeaderSize) + { + state->reqLen = pageHeaderSize; + return true; + } + + /* Now that we know we have the full header, validate it. */ + if (!XLogReaderValidatePageHeader(state, state->readPagePtr, + (char *) state->readBuf)) + { + /* That's bad. Force reading the page again. */ + XLogReaderInvalReadState(state); + + return false; + } + + state->page_verified = true; + + XLByteToSeg(state->readPagePtr, state->seg.ws_segno, + state->segcxt.ws_segsize); + } + + /* + * The loaded page may not be the one caller is supposing to read when we + * are verifying the first page of new segment. In that case, skip further + * verification and immediately load the target page. + */ + if (state->page_verified && pageptr == state->readPagePtr) + { + /* + * calculate additional length for page header keeping the total + * length within the block size. + */ + if (!header_inclusive) + { + uint32 pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + + addLen = pageHeaderSize; + if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + addLen = pageHeaderSize; + else + addLen = XLOG_BLCKSZ - reqLen; + + Assert(addLen >= 0); + } + + /* Return if we already have it. */ + if (reqLen + addLen <= state->readLen) + return false; + } + + /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); + Assert((pageptr % XLOG_BLCKSZ) == 0); - /* check whether we have all the requested data already */ - if (targetSegNo == state->seg.ws_segno && - targetPageOff == state->segoff && reqLen <= state->readLen) - return state->readLen; + /* + * Every time we request to load new data of a page to the caller, even if + * we looked at a part of it before, we need to do verification on the + * next invocation as the caller might now be rereading data from a + * different source. + */ + state->page_verified = false; /* - * Data is not in our buffer. - * - * Every time we actually read the segment, even if we looked at parts of - * it before, we need to do verification as the page_read callback might - * now be rereading data from a different source. - * * Whenever switching to a new WAL segment, we read the first page of the * file and validate its header, even if that's not where the target * record is. This is so that we can check the additional identification - * info that is present in the first page's "long" header. + * info that is present in the first page's "long" header. Don't do this + * if the caller requested the first page in the segment. */ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0) { - XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; - - readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ, - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - - /* we can be sure to have enough WAL available, we scrolled back */ - Assert(readLen == XLOG_BLCKSZ); - - if (!XLogReaderValidatePageHeader(state, targetSegmentPtr, - state->readBuf)) - goto err; + /* + * Then we'll see that the targetSegNo now matches the ws_segno, and + * will not come back here, but will request the actual target page. + */ + state->readPagePtr = pageptr - targetPageOff; + state->reqLen = XLOG_BLCKSZ; + return true; } /* - * First, read the requested data length, but at least a short page header - * so that we can validate it. + * Request the caller to load the page. We need at least a short page + * header so that we can validate it. */ - readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - - Assert(readLen <= XLOG_BLCKSZ); - - /* Do we have enough data to check the header length? */ - if (readLen <= SizeOfXLogShortPHD) - goto err; - - Assert(readLen >= reqLen); - - hdr = (XLogPageHeader) state->readBuf; - - /* still not enough */ - if (readLen < XLogPageHeaderSize(hdr)) - { - readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr), - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - } - - /* - * Now that we know we have the full header, validate it. - */ - if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr)) - goto err; - - /* update read state information */ - state->seg.ws_segno = targetSegNo; - state->segoff = targetPageOff; - state->readLen = readLen; - - return readLen; - -err: - XLogReaderInvalReadState(state); - return -1; + state->readPagePtr = pageptr; + state->reqLen = Max(reqLen + addLen, SizeOfXLogShortPHD); + return true; } /* @@ -676,9 +861,7 @@ err: static void XLogReaderInvalReadState(XLogReaderState *state) { - state->seg.ws_segno = 0; - state->segoff = 0; - state->readLen = 0; + state->readPagePtr = InvalidXLogRecPtr; } /* @@ -686,11 +869,12 @@ XLogReaderInvalReadState(XLogReaderState *state) * * This is just a convenience subroutine to avoid duplicated code in * XLogReadRecord. It's not intended for use from anywhere else. + * + * If PrevRecPtr is valid, the xl_prev is is cross-checked with it. */ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, - XLogRecPtr PrevRecPtr, XLogRecord *record, - bool randAccess) + XLogRecPtr PrevRecPtr, XLogRecord *record) { if (record->xl_tot_len < SizeOfXLogRecord) { @@ -707,7 +891,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, record->xl_rmid, LSN_FORMAT_ARGS(RecPtr)); return false; } - if (randAccess) + if (PrevRecPtr == InvalidXLogRecPtr) { /* * We can't exactly verify the prev-link, but surely it should be less @@ -925,6 +1109,22 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * here. */ +XLogFindNextRecordState * +InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr) +{ + XLogFindNextRecordState *state = (XLogFindNextRecordState *) + palloc_extended(sizeof(XLogFindNextRecordState), + MCXT_ALLOC_NO_OOM | MCXT_ALLOC_ZERO); + if (!state) + return NULL; + + state->reader_state = reader_state; + state->targetRecPtr = start_ptr; + state->currRecPtr = start_ptr; + + return state; +} + /* * Find the first record with an lsn >= RecPtr. * @@ -936,27 +1136,25 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * This positions the reader, like XLogBeginRead(), so that the next call to * XLogReadRecord() will read the next valid record. */ -XLogRecPtr -XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) +bool +XLogFindNextRecord(XLogFindNextRecordState *state) { - XLogRecPtr tmpRecPtr; - XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; + XLogRecord *record; + XLogReadRecordResult result; char *errormsg; - Assert(!XLogRecPtrIsInvalid(RecPtr)); + Assert(!XLogRecPtrIsInvalid(state->currRecPtr)); /* * skip over potential continuation data, keeping in mind that it may span * multiple pages */ - tmpRecPtr = RecPtr; while (true) { XLogRecPtr targetPagePtr; int targetRecOff; uint32 pageHeaderSize; - int readLen; /* * Compute targetRecOff. It should typically be equal or greater than @@ -964,27 +1162,27 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * that, except when caller has explicitly specified the offset that * falls somewhere there or when we are skipping multi-page * continuation record. It doesn't matter though because - * ReadPageInternal() is prepared to handle that and will read at - * least short page-header worth of data + * XLogNeedData() is prepared to handle that and will read at least + * short page-header worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = state->currRecPtr % XLOG_BLCKSZ; /* scroll back to page boundary */ - targetPagePtr = tmpRecPtr - targetRecOff; + targetPagePtr = state->currRecPtr - targetRecOff; - /* Read the page containing the record */ - readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); - if (readLen < 0) + if (XLogNeedData(state->reader_state, targetPagePtr, targetRecOff, + targetRecOff != 0)) + return true; + + if (!state->reader_state->page_verified) goto err; - header = (XLogPageHeader) state->readBuf; + header = (XLogPageHeader) state->reader_state->readBuf; pageHeaderSize = XLogPageHeaderSize(header); - /* make sure we have enough data for the page header */ - readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); - if (readLen < 0) - goto err; + /* we should have read the page header */ + Assert(state->reader_state->readLen >= pageHeaderSize); /* skip over potential continuation data */ if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) @@ -999,21 +1197,21 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * Note that record headers are MAXALIGN'ed */ if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; + state->currRecPtr = targetPagePtr + XLOG_BLCKSZ; else { /* * The previous continuation record ends in this page. Set - * tmpRecPtr to point to the first valid record + * state->currRecPtr to point to the first valid record */ - tmpRecPtr = targetPagePtr + pageHeaderSize + state->currRecPtr = targetPagePtr + pageHeaderSize + MAXALIGN(header->xlp_rem_len); break; } } else { - tmpRecPtr = targetPagePtr + pageHeaderSize; + state->currRecPtr = targetPagePtr + pageHeaderSize; break; } } @@ -1023,31 +1221,36 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * because either we're at the first record after the beginning of a page * or we just jumped over the remaining data of a continuation. */ - XLogBeginRead(state, tmpRecPtr); - while (XLogReadRecord(state, &errormsg) != NULL) + XLogBeginRead(state->reader_state, state->currRecPtr); + while ((result = XLogReadRecord(state->reader_state, &record, &errormsg)) != + XLREAD_FAIL) { + if (result == XLREAD_NEED_DATA) + return true; + /* past the record we've found, break out */ - if (RecPtr <= state->ReadRecPtr) + if (state->targetRecPtr <= state->reader_state->ReadRecPtr) { /* Rewind the reader to the beginning of the last record. */ - found = state->ReadRecPtr; - XLogBeginRead(state, found); - return found; + state->currRecPtr = state->reader_state->ReadRecPtr; + XLogBeginRead(state->reader_state, state->currRecPtr); + return false; } } err: - XLogReaderInvalReadState(state); + XLogReaderInvalReadState(state->reader_state); - return InvalidXLogRecPtr; + state->currRecPtr = InvalidXLogRecPtr;; + return false; } #endif /* FRONTEND */ /* - * Helper function to ease writing of XLogRoutine->page_read callbacks. - * If this function is used, caller must supply a segment_open callback in - * 'state', as that is used here. + * Helper function to ease writing of page_read callback. + * If this function is used, caller must supply a segment_open callback and + * segment_close callback as that is used here. * * Read 'count' bytes into 'buf', starting at location 'startptr', from WAL * fetched from timeline 'tli'. @@ -1060,6 +1263,7 @@ err: */ bool WALRead(XLogReaderState *state, + WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo) { @@ -1091,10 +1295,10 @@ WALRead(XLogReaderState *state, XLogSegNo nextSegNo; if (state->seg.ws_file >= 0) - state->routine.segment_close(state); + segclosefn(state); XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize); - state->routine.segment_open(state, nextSegNo, &tli); + segopenfn(state, nextSegNo, &tli); /* This shouldn't happen -- indicates a bug in segment_open */ Assert(state->seg.ws_file >= 0); diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index d17d660f46..61361192e7 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -686,8 +686,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum, void XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength) { - const XLogRecPtr lastReadPage = (state->seg.ws_segno * - state->segcxt.ws_segsize + state->segoff); + const XLogRecPtr lastReadPage = state->readPagePtr; Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0); Assert(wantLength <= XLOG_BLCKSZ); @@ -702,7 +701,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * current TLI has since become historical. */ if (lastReadPage == wantPage && - state->readLen != 0 && + state->page_verified && lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1)) return; @@ -788,6 +787,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, char path[MAXPGPATH]; XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize); + elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path); state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY); if (state->seg.ws_file >= 0) return; @@ -824,10 +824,12 @@ wal_segment_close(XLogReaderState *state) * exists for normal backends, so we have to do a check/sleep/repeat style of * loop for now. */ -int -read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page) +bool +read_local_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->reqLen; + char *cur_page = state->readBuf; XLogRecPtr read_upto, loc; TimeLineID tli; @@ -926,7 +928,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, else if (targetPagePtr + reqLen > read_upto) { /* not enough data there */ - return -1; + XLogReaderNotifySize(state, -1); + return false; } else { @@ -939,12 +942,14 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, * as 'count', read the whole page anyway. It's guaranteed to be * zero-padded up to the page boundary if it's incomplete. */ - if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli, - &errinfo)) + if (!WALRead(state, wal_segment_open, wal_segment_close, + cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo)) WALReadRaiseError(&errinfo); /* number of valid bytes in the buffer */ - return count; + state->readPagePtr = targetPagePtr; + XLogReaderNotifySize(state, count); + return true; } /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index d61ef4cfad..f1a711a6f3 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options, TransactionId xmin_horizon, bool need_full_snapshot, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx); + ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); ctx->snapshot_builder = @@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin, ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon, need_full_snapshot, false, - xl_routine, prepare_write, do_write, + page_read, cleanup_cb, prepare_write, do_write, update_progress); /* call output plugin initialization callback */ @@ -478,7 +481,8 @@ LogicalDecodingContext * CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -530,8 +534,8 @@ CreateDecodingContext(XLogRecPtr start_lsn, ctx = StartupDecodingContext(output_plugin_options, start_lsn, InvalidTransactionId, false, - fast_forward, xl_routine, prepare_write, - do_write, update_progress); + fast_forward, page_read, cleanup_cb, + prepare_write, do_write, update_progress); /* call output plugin initialization callback */ old_context = MemoryContextSwitchTo(ctx->context); @@ -599,7 +603,13 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - record = XLogReadRecord(ctx->reader, &err); + while (XLogReadRecord(ctx->reader, &record, &err) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + break; + } + if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index 1f38c5b33e..cfc8a02935 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -233,9 +233,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin ctx = CreateDecodingContext(InvalidXLogRecPtr, options, false, - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, LogicalOutputPrepareWrite, LogicalOutputWrite, NULL); @@ -284,7 +283,13 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - record = XLogReadRecord(ctx->reader, &errm); + while (XLogReadRecord(ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + break; + } + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 31e74d3832..bd7f73b8ba 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -153,9 +153,8 @@ create_logical_replication_slot(char *name, char *plugin, ctx = CreateInitDecodingContext(plugin, NIL, false, /* just catalogs is OK */ restart_lsn, - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, NULL, NULL, NULL); /* @@ -512,9 +511,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) ctx = CreateDecodingContext(InvalidXLogRecPtr, NIL, true, /* fast_forward */ - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, NULL, NULL, NULL); /* @@ -536,7 +534,13 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - record = XLogReadRecord(ctx->reader, &errm); + while (XLogReadRecord(ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + break; + } + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 3ca2a11389..a3e350effe 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -575,10 +575,7 @@ StartReplication(StartReplicationCmd *cmd) /* create xlogreader for physical replication */ xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), - NULL); + XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, @@ -802,10 +799,12 @@ StartReplication(StartReplicationCmd *cmd) * which has to do a plain sleep/busy loop, because the walsender's latch gets * set every time WAL is flushed. */ -static int -logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page) +static bool +logical_read_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->reqLen; + char *cur_page = state->readBuf; XLogRecPtr flushptr; int count; WALReadError errinfo; @@ -822,7 +821,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* fail if not (implies we are going to shut down) */ if (flushptr < targetPagePtr + reqLen) - return -1; + { + XLogReaderNotifySize(state, -1); + return false; + } if (targetPagePtr + XLOG_BLCKSZ <= flushptr) count = XLOG_BLCKSZ; /* more than one block available */ @@ -830,7 +832,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req count = flushptr - targetPagePtr; /* part of the page available */ /* now actually read the data, we know it's there */ - if (!WALRead(state, + if (!WALRead(state, WalSndSegmentOpen, wal_segment_close, cur_page, targetPagePtr, XLOG_BLCKSZ, @@ -850,7 +852,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize); CheckXLogRemoved(segno, state->seg.ws_tli); - return count; + XLogReaderNotifySize(state, count); + return true; } /* @@ -1015,9 +1018,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot, InvalidXLogRecPtr, - XL_ROUTINE(.page_read = logical_read_xlog_page, - .segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), + logical_read_xlog_page, + wal_segment_close, WalSndPrepareWrite, WalSndWriteData, WalSndUpdateProgress); @@ -1175,9 +1177,8 @@ StartLogicalReplication(StartReplicationCmd *cmd) */ logical_decoding_ctx = CreateDecodingContext(cmd->startpoint, cmd->options, false, - XL_ROUTINE(.page_read = logical_read_xlog_page, - .segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), + logical_read_xlog_page, + wal_segment_close, WalSndPrepareWrite, WalSndWriteData, WalSndUpdateProgress); xlogreader = logical_decoding_ctx->reader; @@ -2770,7 +2771,7 @@ XLogSendPhysical(void) enlargeStringInfo(&output_message, nbytes); retry: - if (!WALRead(xlogreader, + if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close, &output_message.data[output_message.len], startptr, nbytes, @@ -2868,7 +2869,12 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - record = XLogReadRecord(logical_decoding_ctx->reader, &errm); + while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader)) + break; + } /* xlog record was invalid */ if (errm != NULL) diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 59ebac7d6a..707493dddf 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -41,15 +41,9 @@ static int xlogreadfd = -1; static XLogSegNo xlogreadsegno = -1; static char xlogfpath[MAXPGPATH]; -typedef struct XLogPageReadPrivate -{ - const char *restoreCommand; - int tliIndex; -} XLogPageReadPrivate; - -static int SimpleXLogPageRead(XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf); +static bool SimpleXLogPageRead(XLogReaderState *xlogreader, + const char *datadir, int *tliIndex, + const char *restoreCommand); /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline @@ -66,20 +60,22 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + if (xlogreader == NULL) pg_fatal("out of memory"); XLogBeginRead(xlogreader, startpoint); do { - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, + &tliIndex, restoreCommand)) + break; + } if (record == NULL) { @@ -123,19 +119,19 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; XLogRecPtr endptr; - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); XLogBeginRead(xlogreader, ptr); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand)) + break; + } if (record == NULL) { if (errormsg) @@ -170,7 +166,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr searchptr; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; /* * The given fork pointer points to the end of the last common record, @@ -186,11 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -200,7 +191,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, uint8 info; XLogBeginRead(xlogreader, searchptr); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, + &tliIndex, restoreCommand)) + break; + } if (record == NULL) { @@ -246,16 +243,19 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, } /* XLogReader callback function, to read a WAL page */ -static int -SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf) +static bool +SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, + int *tliIndex, const char *restoreCommand) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogRecPtr targetPagePtr = xlogreader->readPagePtr; + char *readBuf = xlogreader->readBuf; uint32 targetPageOff; XLogRecPtr targetSegEnd; XLogSegNo targetSegNo; int r; + Assert(xlogreader->reqLen <= XLOG_BLCKSZ); + XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz); XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd); targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz); @@ -283,14 +283,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, * be done both forward and backward, consider also switching timeline * accordingly. */ - while (private->tliIndex < targetNentries - 1 && - targetHistory[private->tliIndex].end < targetSegEnd) - private->tliIndex++; - while (private->tliIndex > 0 && - targetHistory[private->tliIndex].begin >= targetSegEnd) - private->tliIndex--; + while (*tliIndex < targetNentries - 1 && + targetHistory[*tliIndex].end < targetSegEnd) + (*tliIndex)++; + while (*tliIndex > 0 && + targetHistory[*tliIndex].begin >= targetSegEnd) + (*tliIndex)--; - XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, + XLogFileName(xlogfname, targetHistory[*tliIndex].tli, xlogreadsegno, WalSegSz); snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", @@ -303,10 +303,11 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* * If we have no restore_command to execute, then exit. */ - if (private->restoreCommand == NULL) + if (restoreCommand == NULL) { pg_log_error("could not open file \"%s\": %m", xlogfpath); - return -1; + XLogReaderNotifySize(xlogreader, -1); + return false; } /* @@ -316,10 +317,13 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir, xlogfname, WalSegSz, - private->restoreCommand); + restoreCommand); if (xlogreadfd < 0) - return -1; + { + XLogReaderNotifySize(xlogreader, -1); + return false; + } else pg_log_debug("using file \"%s\" restored from archive", xlogfpath); @@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) { pg_log_error("could not seek in file \"%s\": %m", xlogfpath); - return -1; + XLogReaderNotifySize(xlogreader, -1); + return false; } @@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, pg_log_error("could not read file \"%s\": read %d of %zu", xlogfpath, r, (Size) XLOG_BLCKSZ); - return -1; + XLogReaderNotifySize(xlogreader, -1); + return false; } Assert(targetSegNo == xlogreadsegno); - xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli; - return XLOG_BLCKSZ; + xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli; + XLogReaderNotifySize(xlogreader, XLOG_BLCKSZ); + return true; } /* diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 74664bef6a..12dcbf1685 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -29,14 +29,6 @@ static const char *progname; static int WalSegSz; -typedef struct XLogDumpPrivate -{ - TimeLineID timeline; - XLogRecPtr startptr; - XLogRecPtr endptr; - bool endptr_reached; -} XLogDumpPrivate; - typedef struct XLogDumpConfig { /* display options */ @@ -331,30 +323,41 @@ WALDumpCloseSegment(XLogReaderState *state) state->seg.ws_file = -1; } -/* pg_waldump's XLogReaderRoutine->page_read callback */ -static int -WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetPtr, char *readBuff) +/* + * pg_waldump's WAL page rader + * + * timeline and startptr specifies the LSN, and reads up to endptr. + */ +static bool +WALDumpReadPage(XLogReaderState *state, TimeLineID timeline, + XLogRecPtr startptr, XLogRecPtr endptr) { - XLogDumpPrivate *private = state->private_data; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->reqLen; + char *readBuff = state->readBuf; int count = XLOG_BLCKSZ; WALReadError errinfo; - if (private->endptr != InvalidXLogRecPtr) + /* determine the number of bytes to read on the page */ + if (endptr != InvalidXLogRecPtr) { - if (targetPagePtr + XLOG_BLCKSZ <= private->endptr) + if (targetPagePtr + XLOG_BLCKSZ <= endptr) count = XLOG_BLCKSZ; - else if (targetPagePtr + reqLen <= private->endptr) - count = private->endptr - targetPagePtr; + else if (targetPagePtr + reqLen <= endptr) + count = endptr - targetPagePtr; else { - private->endptr_reached = true; - return -1; + /* Notify xlogreader that we didn't read at all */ + XLogReaderNotifySize(state, -1); + return false; } } - if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline, - &errinfo)) + /* We should read more than requested by xlogreader */ + Assert(count >= state->readLen); + + if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment, + readBuff, targetPagePtr, count, timeline, &errinfo)) { WALOpenSegment *seg = &errinfo.wre_seg; char fname[MAXPGPATH]; @@ -374,7 +377,9 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, (Size) errinfo.wre_req); } - return count; + /* Notify xlogreader of how many bytes we have read */ + XLogReaderNotifySize(state, count); + return true; } /* @@ -766,7 +771,10 @@ main(int argc, char **argv) uint32 xlogid; uint32 xrecoff; XLogReaderState *xlogreader_state; - XLogDumpPrivate private; + XLogFindNextRecordState *findnext_state; + TimeLineID timeline; + XLogRecPtr startptr; + XLogRecPtr endptr; XLogDumpConfig config; XLogDumpStats stats; XLogRecord *record; @@ -812,14 +820,9 @@ main(int argc, char **argv) } } - memset(&private, 0, sizeof(XLogDumpPrivate)); - memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); - - private.timeline = 1; - private.startptr = InvalidXLogRecPtr; - private.endptr = InvalidXLogRecPtr; - private.endptr_reached = false; + timeline = 1; + startptr = InvalidXLogRecPtr; + endptr = InvalidXLogRecPtr; config.quiet = false; config.bkp_details = false; @@ -854,7 +857,7 @@ main(int argc, char **argv) optarg); goto bad_argument; } - private.endptr = (uint64) xlogid << 32 | xrecoff; + endptr = (uint64) xlogid << 32 | xrecoff; break; case 'f': config.follow = true; @@ -907,10 +910,10 @@ main(int argc, char **argv) goto bad_argument; } else - private.startptr = (uint64) xlogid << 32 | xrecoff; + startptr = (uint64) xlogid << 32 | xrecoff; break; case 't': - if (sscanf(optarg, "%d", &private.timeline) != 1) + if (sscanf(optarg, "%d", &timeline) != 1) { pg_log_error("could not parse timeline \"%s\"", optarg); goto bad_argument; @@ -987,21 +990,21 @@ main(int argc, char **argv) close(fd); /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &segno, WalSegSz); + XLogFromFileName(fname, &timeline, &segno, WalSegSz); - if (XLogRecPtrIsInvalid(private.startptr)) - XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, private.startptr); - else if (!XLByteInSeg(private.startptr, segno, WalSegSz)) + if (XLogRecPtrIsInvalid(startptr)) + XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, startptr); + else if (!XLByteInSeg(startptr, segno, WalSegSz)) { pg_log_error("start WAL location %X/%X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.startptr), + LSN_FORMAT_ARGS(startptr), fname); goto bad_argument; } /* no second file specified, set end position */ - if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(private.endptr)) - XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, private.endptr); + if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(endptr)) + XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, endptr); /* parse ENDSEG if passed */ if (optind + 1 < argc) @@ -1017,26 +1020,26 @@ main(int argc, char **argv) close(fd); /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz); + XLogFromFileName(fname, &timeline, &endsegno, WalSegSz); if (endsegno < segno) fatal_error("ENDSEG %s is before STARTSEG %s", argv[optind + 1], argv[optind]); - if (XLogRecPtrIsInvalid(private.endptr)) + if (XLogRecPtrIsInvalid(endptr)) XLogSegNoOffsetToRecPtr(endsegno + 1, 0, WalSegSz, - private.endptr); + endptr); /* set segno to endsegno for check of --end */ segno = endsegno; } - if (!XLByteInSeg(private.endptr, segno, WalSegSz) && - private.endptr != (segno + 1) * WalSegSz) + if (!XLByteInSeg(endptr, segno, WalSegSz) && + endptr != (segno + 1) * WalSegSz) { pg_log_error("end WAL location %X/%X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.endptr), + LSN_FORMAT_ARGS(endptr), argv[argc - 1]); goto bad_argument; } @@ -1045,7 +1048,7 @@ main(int argc, char **argv) waldir = identify_target_directory(waldir, NULL); /* we don't know what to print */ - if (XLogRecPtrIsInvalid(private.startptr)) + if (XLogRecPtrIsInvalid(startptr)) { pg_log_error("no start WAL location given"); goto bad_argument; @@ -1055,42 +1058,56 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, - XL_ROUTINE(.page_read = WALDumpReadPage, - .segment_open = WALDumpOpenSegment, - .segment_close = WALDumpCloseSegment), - &private); + XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + if (!xlogreader_state) fatal_error("out of memory"); + findnext_state = + InitXLogFindNextRecord(xlogreader_state, startptr); + + if (!findnext_state) + fatal_error("out of memory"); + /* first find a valid recptr to start from */ - first_record = XLogFindNextRecord(xlogreader_state, private.startptr); + while (XLogFindNextRecord(findnext_state)) + { + if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr)) + break; + } + first_record = findnext_state->currRecPtr; if (first_record == InvalidXLogRecPtr) fatal_error("could not find a valid record after %X/%X", - LSN_FORMAT_ARGS(private.startptr)); + LSN_FORMAT_ARGS(startptr)); /* * Display a message that we're skipping data if `from` wasn't a pointer * to the start of a record and also wasn't a pointer to the beginning of * a segment (e.g. we were used in file mode). */ - if (first_record != private.startptr && - XLogSegmentOffset(private.startptr, WalSegSz) != 0) + if (first_record != startptr && + XLogSegmentOffset(startptr, WalSegSz) != 0) printf(ngettext("first record is after %X/%X, at %X/%X, skipping over %u byte\n", "first record is after %X/%X, at %X/%X, skipping over %u bytes\n", - (first_record - private.startptr)), - LSN_FORMAT_ARGS(private.startptr), + (first_record - startptr)), + LSN_FORMAT_ARGS(startptr), LSN_FORMAT_ARGS(first_record), - (uint32) (first_record - private.startptr)); + (uint32) (first_record - startptr)); for (;;) { /* try to read the next record */ - record = XLogReadRecord(xlogreader_state, &errormsg); + while (XLogReadRecord(xlogreader_state, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr)) + break; + } + if (!record) { - if (!config.follow || private.endptr_reached) + if (!config.follow) break; else { diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 21d200d3df..836ca7fce8 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -56,65 +56,17 @@ typedef struct WALSegmentContext } WALSegmentContext; typedef struct XLogReaderState XLogReaderState; +typedef struct XLogFindNextRecordState XLogFindNextRecordState; -/* Function type definitions for various xlogreader interactions */ -typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, - XLogRecPtr targetRecPtr, - char *readBuf); +/* Function type definition for the segment cleanup callback */ +typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader); + +/* Function type definition for the open/close callbacks for WALRead() */ typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p); typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader); -typedef struct XLogReaderRoutine -{ - /* - * Data input callback - * - * This callback shall read at least reqLen valid bytes of the xlog page - * starting at targetPagePtr, and store them in readBuf. The callback - * shall return the number of bytes read (never more than XLOG_BLCKSZ), or - * -1 on failure. The callback shall sleep, if necessary, to wait for the - * requested bytes to become available. The callback will not be invoked - * again for the same page unless more than the returned number of bytes - * are needed. - * - * targetRecPtr is the position of the WAL record we're reading. Usually - * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs - * to read and verify the page or segment header, before it reads the - * actual WAL record it's interested in. In that case, targetRecPtr can - * be used to determine which timeline to read the page from. - * - * The callback shall set ->seg.ws_tli to the TLI of the file the page was - * read from. - */ - XLogPageReadCB page_read; - - /* - * Callback to open the specified WAL segment for reading. ->seg.ws_file - * shall be set to the file descriptor of the opened segment. In case of - * failure, an error shall be raised by the callback and it shall not - * return. - * - * "nextSegNo" is the number of the segment to be opened. - * - * "tli_p" is an input/output argument. WALRead() uses it to pass the - * timeline in which the new segment should be found, but the callback can - * use it to return the TLI that it actually opened. - */ - WALSegmentOpenCB segment_open; - - /* - * WAL segment close callback. ->seg.ws_file shall be set to a negative - * number. - */ - WALSegmentCloseCB segment_close; -} XLogReaderRoutine; - -#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__} - typedef struct { /* Is this block ref in use? */ @@ -144,12 +96,36 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* Return code from XLogReadRecord */ +typedef enum XLogReadRecordResult +{ + XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */ + XLREAD_FAIL /* failed during reading a record */ +} XLogReadRecordResult; + +/* + * internal state of XLogReadRecord + * + * XLogReadState runs a state machine while reading a record. Theses states + * are not seen outside the function. Each state may repeat several times + * exiting requesting caller for new data. See the comment of XLogReadRecrod + * for details. + */ +typedef enum XLogReadRecordState +{ + XLREAD_NEXT_RECORD, + XLREAD_TOT_LEN, + XLREAD_FIRST_FRAGMENT, + XLREAD_CONTINUATION +} XLogReadRecordState; + struct XLogReaderState { /* * Operational callbacks */ - XLogReaderRoutine routine; + WALSegmentCleanupCB cleanup_cb; /* ---------------------------------------- * Public parameters @@ -162,19 +138,32 @@ struct XLogReaderState */ uint64 system_identifier; - /* - * Opaque data for callbacks to use. Not used by XLogReader. - */ - void *private_data; - /* * Start and end point of last record read. EndRecPtr is also used as the * position to read next. Calling XLogBeginRead() sets EndRecPtr to the * starting position and ReadRecPtr to invalid. */ - XLogRecPtr ReadRecPtr; /* start of last record read */ + XLogRecPtr ReadRecPtr; /* start of last record read or being read */ XLogRecPtr EndRecPtr; /* end+1 of last record read */ + XLogRecPtr PrevRecPtr; /* start of previous record read */ + /* ---------------------------------------- + * Communication with page reader + * readBuf is XLOG_BLCKSZ bytes, valid up to at least reqLen bytes. + * ---------------------------------------- + */ + /* variables to inform to the callers from xlogreader */ + XLogRecPtr readPagePtr; /* page pointer to read */ + int32 reqLen; /* bytes requested to the caller */ + char *readBuf; /* buffer to store data */ + bool page_verified; /* is the page header on the buffer verified? */ + bool record_verified;/* is the current record header verified? */ + + /* variables to respond from the callers to xlogreader */ + int32 readLen; /* actual bytes read by reader, which must be + * larger than the request, or -1 on error. + * Use XLogReaderNotifyLength() to set a + * value. */ /* ---------------------------------------- * Decoded representation of current record @@ -203,13 +192,6 @@ struct XLogReaderState * ---------------------------------------- */ - /* - * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least - * readLen bytes) - */ - char *readBuf; - uint32 readLen; - /* last read XLOG position for data currently in readBuf */ WALSegmentContext segcxt; WALOpenSegment seg; @@ -222,8 +204,6 @@ struct XLogReaderState XLogRecPtr latestPagePtr; TimeLineID latestPageTLI; - /* beginning of the WAL record being read. */ - XLogRecPtr currRecPtr; /* timeline to read it from, 0 if a lookup is required */ TimeLineID currTLI; @@ -250,16 +230,37 @@ struct XLogReaderState char *readRecordBuf; uint32 readRecordBufSize; + /* + * XLogReadRecord() state + */ + XLogReadRecordState readRecordState; /* state machine state */ + int recordGotLen; /* amount of current record that has already + * been read */ + int recordRemainLen; /* length of current record that remains */ + XLogRecPtr recordContRecPtr; /* where the current record continues */ + /* Buffer to hold error message */ char *errormsg_buf; }; +struct XLogFindNextRecordState +{ + XLogReaderState *reader_state; + XLogRecPtr targetRecPtr; + XLogRecPtr currRecPtr; +}; + +/* setter functions of XLogReaderState used by other modules */ +static inline void +XLogReaderNotifySize(XLogReaderState *state, int32 len) +{ + state->readLen = len; +} + /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, const char *waldir, - XLogReaderRoutine *routine, - void *private_data); -extern XLogReaderRoutine *LocalXLogReaderRoutine(void); + WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ extern void XLogReaderFree(XLogReaderState *state); @@ -267,12 +268,17 @@ extern void XLogReaderFree(XLogReaderState *state); /* Position the XLogReader to given record */ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr); #ifdef FRONTEND -extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); +/* Function type definition for the read_page callback */ +typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader, + void *private); +extern XLogFindNextRecordState *InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr); +extern bool XLogFindNextRecord(XLogFindNextRecordState *state); #endif /* FRONTEND */ /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ -extern struct XLogRecord *XLogReadRecord(XLogReaderState *state, - char **errormsg); +extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state, + XLogRecord **record, + char **errormsg); /* Validate a page */ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, @@ -292,6 +298,7 @@ typedef struct WALReadError } WALReadError; extern bool WALRead(XLogReaderState *state, + WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo); diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 9ac602b674..397fb27fc2 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -47,9 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern int read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page); +extern bool read_local_xlog_page(XLogReaderState *state); extern void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index 614035e215..fecffdb3f6 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -390,7 +390,7 @@ * Enable debugging print statements for WAL-related operations; see * also the wal_debug GUC var. */ -/* #define WAL_DEBUG */ +#define WAL_DEBUG /* * Enable tracing of resource consumption during sort operations; diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index e0f513b773..3346188b48 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC TransactionId xid ); +typedef struct LogicalDecodingContext LogicalDecodingContext; + +typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx); + typedef struct LogicalDecodingContext { /* memory context this is all allocated in */ @@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext /* infrastructure pieces for decoding */ XLogReaderState *reader; + LogicalDecodingXLogPageReadCB page_read; struct ReorderBuffer *reorder; struct SnapBuild *snapshot_builder; @@ -115,14 +120,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); -- 2.27.0
On Thu, Jul 15, 2021 at 4:48 PM Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote: > Gah... Thank you for noticing me. I thought that I have sent the > rebased version. This is the rebased version on the current master. Hi Kyotaro, Did you see this? https://www.postgresql.org/message-id/20210429022553.4h5qii5jb5eclu4i%40alap3.anarazel.de
At Mon, 27 Sep 2021 17:31:03 +1300, Thomas Munro <thomas.munro@gmail.com> wrote in > On Thu, Jul 15, 2021 at 4:48 PM Kyotaro Horiguchi > <horikyota.ntt@gmail.com> wrote: > > Gah... Thank you for noticing me. I thought that I have sent the > > rebased version. This is the rebased version on the current master. > > Hi Kyotaro, > > Did you see this? > > https://www.postgresql.org/message-id/20210429022553.4h5qii5jb5eclu4i%40alap3.anarazel.de Thank you for pinging me. I haven't noticed of that. I'll check on that line. regards. -- Kyotaro Horiguchi NTT Open Source Software Center
At Thu, 30 Sep 2021 09:40:06 +0900 (JST), Kyotaro Horiguchi <horikyota.ntt@gmail.com> wrote in > At Mon, 27 Sep 2021 17:31:03 +1300, Thomas Munro <thomas.munro@gmail.com> wrote in > > On Thu, Jul 15, 2021 at 4:48 PM Kyotaro Horiguchi > > <horikyota.ntt@gmail.com> wrote: > > > Gah... Thank you for noticing me. I thought that I have sent the > > > rebased version. This is the rebased version on the current master. > > > > Hi Kyotaro, > > > > Did you see this? > > > > https://www.postgresql.org/message-id/20210429022553.4h5qii5jb5eclu4i%40alap3.anarazel.de > > Thank you for pinging me. I haven't noticed of that. > I'll check on that line. It looks like the XLogFindNextRecord was not finished. It should have been turned into a state machine. In this version (v18), This contains only page-reader refactoring stuff. - Rebased to the current master, including additional change for XLOG_OVERWRITE_CONTRECORD stuff. (This needed the new function XLogTerminateRead.) - Finished XLogFindNextRecord including the fixup from Thomas' v17. - Added a test for XLogFindNextRecord, on the behavior that page-skipping on seeking for the first record. regards. -- Kyotaro Horiguchi NTT Open Source Software Center From 2da06a9de78e4f2125bcc889d12e7e3ffdf1d4d4 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Thu, 30 Sep 2021 11:48:40 +0900 Subject: [PATCH v18 1/5] Move callback-call from ReadPageInternal to XLogReadRecord. The current WAL record reader reads page data using a call back function. Redesign the interface so that it asks the caller for more data when required. This model works better for proposed projects that encryption, prefetching and other new features that would require extending the callback interface for each case. As the first step of that change, this patch moves the page reader function out of ReadPageInternal(), then the remaining tasks of the function are taken over by the new function XLogNeedData(). Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> Author: Heikki Linnakangas <hlinnaka@iki.fi> Reviewed-by: Antonin Houska <ah@cybertec.at> Reviewed-by: Alvaro Herrera <alvherre@2ndquadrant.com> Reviewed-by: Takashi Menjo <takashi.menjo@gmail.com> Reviewed-by: Thomas Munro <thomas.munro@gmail.com> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/xlog.c | 16 +- src/backend/access/transam/xlogreader.c | 325 +++++++++++++++--------- src/backend/access/transam/xlogutils.c | 12 +- src/backend/replication/walsender.c | 10 +- src/bin/pg_rewind/parsexlog.c | 21 +- src/bin/pg_waldump/pg_waldump.c | 8 +- src/include/access/xlogreader.h | 31 ++- src/include/access/xlogutils.h | 2 +- 8 files changed, 266 insertions(+), 159 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 26dcc00ac0..1557ceb8c1 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -918,7 +918,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, +static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, bool fetching_ckpt, XLogRecPtr tliRecPtr); @@ -4401,7 +4401,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode, XLogRecord *record; XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - /* Pass through parameters to XLogPageRead */ private->fetching_ckpt = fetching_ckpt; private->emode = emode; private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr); @@ -12300,7 +12299,7 @@ CancelBackup(void) * XLogPageRead() to try fetching the record from another source, or to * sleep and retry. */ -static int +static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf) { @@ -12359,7 +12358,8 @@ retry: readLen = 0; readSource = XLOG_FROM_ANY; - return -1; + xlogreader->readLen = -1; + return false; } } @@ -12469,7 +12469,8 @@ retry: goto next_record_is_invalid; } - return readLen; + xlogreader->readLen = readLen; + return true; next_record_is_invalid: lastSourceFailed = true; @@ -12483,8 +12484,9 @@ next_record_is_invalid: /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - else - return -1; + + xlogreader->readLen = -1; + return false; } /* diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 4b03577dcc..1d9976ecf4 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -39,8 +39,8 @@ static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); -static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, - int reqLen); +static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, + int reqLen, bool header_inclusive); static void XLogReaderInvalReadState(XLogReaderState *state); static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); @@ -264,8 +264,48 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) * If the reading fails for some other reason, NULL is also returned, and * *errormsg is set to a string with details of the failure. * + * Returns XLREAD_NEED_DATA if more data is needed to finish reading the + * current record. In that case, state->readPagePtr and state->readLen inform + * the desired position and minimum length of data needed. The caller shall + * read in the requested data and set state->readBuf to point to a buffer + * containing it. The caller must also set state->seg->ws_tli and + * state->readLen to indicate the timeline that it was read from, and the + * length of data that is now available (which must be >= given readLen), + * respectively. + * + * If invalid data is encountered, returns XLREAD_FAIL with *record being set to + * NULL. *errormsg is set to a string with details of the failure. * The returned pointer (or *errormsg) points to an internal buffer that's * valid until the next call to XLogReadRecord. + * + * + * This function runs a state machine consists of the following states. + * + * XLREAD_NEXT_RECORD : + * The initial state, if called with valid RecPtr, try to read a record at + * that position. If invalid RecPtr is given try to read a record just after + * the last one previously read. + * This state ens after setting ReadRecPtr. Then goes to XLREAD_TOT_LEN. + * + * XLREAD_TOT_LEN: + * Examining record header. Ends after reading record total + * length. recordRemainLen and recordGotLen are initialized. + * + * XLREAD_FIRST_FRAGMENT: + * Reading the first fragment. Ends with finishing reading a single + * record. Goes to XLREAD_NEXT_RECORD if that's all or + * XLREAD_CONTINUATION if we have continuation. + + * XLREAD_CONTINUATION: + * Reading continuation of record. Ends with finishing the whole record then + * goes to XLREAD_NEXT_RECORD. During this state, recordRemainLen indicates + * how much is left and readRecordBuf holds the partially assert + * record.recordContRecPtr points to the beginning of the next page where to + * continue. + * + * If wrong data found in any state, the state machine stays at the current + * state. This behavior allows to continue reading a reacord switching among + * different souces, while streaming replication. */ XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg) @@ -280,7 +320,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) uint32 pageHeaderSize; bool assembled; bool gotheader; - int readOff; /* * randAccess indicates whether to verify the previous-record pointer of @@ -334,14 +373,20 @@ restart: * byte to cover the whole record header, or at least the part of it that * fits on the same page. */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + targetRecOff != 0)) + { + if (!state->routine.page_read(state, state->readPagePtr, state->readLen, + RecPtr, state->readBuf)) + break; + } + + if (!state->page_verified) goto err; /* - * ReadPageInternal always returns at least the page header, so we can - * examine it now. + * We have at least the page header, so we can examine it now. */ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); if (targetRecOff == 0) @@ -367,8 +412,8 @@ restart: goto err; } - /* ReadPageInternal has verified the page header */ - Assert(pageHeaderSize <= readOff); + /* XLogNeedData has verified the page header */ + Assert(pageHeaderSize <= state->readLen); /* * Read the record length. @@ -442,18 +487,27 @@ restart: do { + int rest_len = total_len - gotlen; + /* Calculate pointer to beginning of next page */ targetPagePtr += XLOG_BLCKSZ; /* Wait for the next page to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(total_len - gotlen + SizeOfXLogShortPHD, - XLOG_BLCKSZ)); + while (XLogNeedData(state, targetPagePtr, + Min(rest_len, XLOG_BLCKSZ), + false)) + { + if (!state->routine.page_read(state, state->readPagePtr, + state->readLen, + state->ReadRecPtr, + state->readBuf)) + break; + } - if (readOff < 0) + if (!state->page_verified) goto err; - Assert(SizeOfXLogShortPHD <= readOff); + Assert(SizeOfXLogShortPHD <= state->readLen); pageHeader = (XLogPageHeader) state->readBuf; @@ -500,21 +554,14 @@ restart: /* Append the continuation from this page to the buffer */ pageHeaderSize = XLogPageHeaderSize(pageHeader); - if (readOff < pageHeaderSize) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize); - - Assert(pageHeaderSize <= readOff); + Assert(pageHeaderSize <= state->readLen); contdata = (char *) state->readBuf + pageHeaderSize; len = XLOG_BLCKSZ - pageHeaderSize; if (pageHeader->xlp_rem_len < len) len = pageHeader->xlp_rem_len; - if (readOff < pageHeaderSize + len) - readOff = ReadPageInternal(state, targetPagePtr, - pageHeaderSize + len); - + Assert(pageHeaderSize + len <= state->readLen); memcpy(buffer, (char *) contdata, len); buffer += len; gotlen += len; @@ -544,9 +591,16 @@ restart: else { /* Wait for the record data to become available */ - readOff = ReadPageInternal(state, targetPagePtr, - Min(targetRecOff + total_len, XLOG_BLCKSZ)); - if (readOff < 0) + while (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + total_len, XLOG_BLCKSZ), true)) + { + if (!state->routine.page_read(state, state->readPagePtr, + state->readLen, + state->ReadRecPtr, state->readBuf)) + break; + } + + if (!state->page_verified) goto err; /* Record does not cross a page boundary */ @@ -603,109 +657,138 @@ err: } /* - * Read a single xlog page including at least [pageptr, reqLen] of valid data - * via the page_read() callback. + * Checks that an xlog page loaded in state->readBuf is including at least + * [pageptr, reqLen] and the page is valid. header_inclusive indicates that + * reqLen is calculated including page header length. * - * Returns -1 if the required page cannot be read for some reason; errormsg_buf - * is set in that case (unless the error occurs in the page_read callback). + * Returns false if the buffer already contains the requested data, or found + * error. state->page_verified is set to true for the former and false for the + * latter. * - * We fetch the page from a reader-local cache if we know we have the required - * data and if there hasn't been any error since caching the data. + * Otherwise returns true and requests data loaded onto state->readBuf by + * state->readPagePtr and state->readLen. The caller shall call this function + * again after filling the buffer at least with that portion of data and set + * state->readLen to the length of actually loaded data. + * + * If header_inclusive is false, corrects reqLen internally by adding the + * actual page header length and may request caller for new data. */ -static int -ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) +static bool +XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, + bool header_inclusive) { - int readLen; uint32 targetPageOff; XLogSegNo targetSegNo; - XLogPageHeader hdr; + uint32 addLen = 0; - Assert((pageptr % XLOG_BLCKSZ) == 0); + /* Some data is loaded, but page header is not verified yet. */ + if (!state->page_verified && + !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0) + { + uint32 pageHeaderSize; + /* just loaded new data so needs to verify page header */ + + /* The caller must have loaded at least page header */ + Assert(state->readLen >= SizeOfXLogShortPHD); + + /* + * We have enough data to check the header length. Recheck the loaded + * length against the actual header length. + */ + pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); + + /* Request more data if we don't have the full header. */ + if (state->readLen < pageHeaderSize) + { + state->readLen = pageHeaderSize; + return true; + } + + /* Now that we know we have the full header, validate it. */ + if (!XLogReaderValidatePageHeader(state, state->readPagePtr, + (char *) state->readBuf)) + { + /* That's bad. Force reading the page again. */ + XLogReaderInvalReadState(state); + + return false; + } + + state->page_verified = true; + + XLByteToSeg(state->readPagePtr, state->seg.ws_segno, + state->segcxt.ws_segsize); + } + + /* + * The loaded page may not be the one caller is supposing to read when we + * are verifying the first page of new segment. In that case, skip further + * verification and immediately load the target page. + */ + if (state->page_verified && pageptr == state->readPagePtr) + { + /* + * calculate additional length for page header keeping the total + * length within the block size. + */ + if (!header_inclusive) + { + uint32 pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + + addLen = pageHeaderSize; + if (reqLen + pageHeaderSize <= XLOG_BLCKSZ) + addLen = pageHeaderSize; + else + addLen = XLOG_BLCKSZ - reqLen; + + Assert(addLen >= 0); + } + + /* Return if we already have it. */ + if (reqLen + addLen <= state->readLen) + return false; + } + + /* Data is not in our buffer, request the caller for it. */ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize); targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize); + Assert((pageptr % XLOG_BLCKSZ) == 0); - /* check whether we have all the requested data already */ - if (targetSegNo == state->seg.ws_segno && - targetPageOff == state->segoff && reqLen <= state->readLen) - return state->readLen; + /* + * Every time we request to load new data of a page to the caller, even if + * we looked at a part of it before, we need to do verification on the + * next invocation as the caller might now be rereading data from a + * different source. + */ + state->page_verified = false; /* - * Data is not in our buffer. - * - * Every time we actually read the segment, even if we looked at parts of - * it before, we need to do verification as the page_read callback might - * now be rereading data from a different source. - * * Whenever switching to a new WAL segment, we read the first page of the * file and validate its header, even if that's not where the target * record is. This is so that we can check the additional identification - * info that is present in the first page's "long" header. + * info that is present in the first page's "long" header. Don't do this + * if the caller requested the first page in the segment. */ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0) { - XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; - - readLen = state->routine.page_read(state, targetSegmentPtr, XLOG_BLCKSZ, - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - - /* we can be sure to have enough WAL available, we scrolled back */ - Assert(readLen == XLOG_BLCKSZ); - - if (!XLogReaderValidatePageHeader(state, targetSegmentPtr, - state->readBuf)) - goto err; + /* + * Then we'll see that the targetSegNo now matches the ws_segno, and + * will not come back here, but will request the actual target page. + */ + state->readPagePtr = pageptr - targetPageOff; + state->readLen = XLOG_BLCKSZ; + return true; } /* - * First, read the requested data length, but at least a short page header - * so that we can validate it. + * Request the caller to load the page. We need at least a short page + * header so that we can validate it. */ - readLen = state->routine.page_read(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - - Assert(readLen <= XLOG_BLCKSZ); - - /* Do we have enough data to check the header length? */ - if (readLen <= SizeOfXLogShortPHD) - goto err; - - Assert(readLen >= reqLen); - - hdr = (XLogPageHeader) state->readBuf; - - /* still not enough */ - if (readLen < XLogPageHeaderSize(hdr)) - { - readLen = state->routine.page_read(state, pageptr, XLogPageHeaderSize(hdr), - state->currRecPtr, - state->readBuf); - if (readLen < 0) - goto err; - } - - /* - * Now that we know we have the full header, validate it. - */ - if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr)) - goto err; - - /* update read state information */ - state->seg.ws_segno = targetSegNo; - state->segoff = targetPageOff; - state->readLen = readLen; - - return readLen; - -err: - XLogReaderInvalReadState(state); - return -1; + state->readPagePtr = pageptr; + state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD); + return true; } /* @@ -714,9 +797,7 @@ err: static void XLogReaderInvalReadState(XLogReaderState *state) { - state->seg.ws_segno = 0; - state->segoff = 0; - state->readLen = 0; + state->readPagePtr = InvalidXLogRecPtr; } /* @@ -994,7 +1075,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) XLogRecPtr targetPagePtr; int targetRecOff; uint32 pageHeaderSize; - int readLen; /* * Compute targetRecOff. It should typically be equal or greater than @@ -1002,27 +1082,32 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * that, except when caller has explicitly specified the offset that * falls somewhere there or when we are skipping multi-page * continuation record. It doesn't matter though because - * ReadPageInternal() is prepared to handle that and will read at - * least short page-header worth of data + * XLogNeedData() is prepared to handle that and will read at least + * short page-header worth of data */ targetRecOff = tmpRecPtr % XLOG_BLCKSZ; /* scroll back to page boundary */ targetPagePtr = tmpRecPtr - targetRecOff; - /* Read the page containing the record */ - readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); - if (readLen < 0) + while (XLogNeedData(state, targetPagePtr, targetRecOff, + targetRecOff != 0)) + { + if (!state->routine.page_read(state, state->readPagePtr, + state->readLen, + state->ReadRecPtr, state->readBuf)) + break; + } + + if (!state->page_verified) goto err; header = (XLogPageHeader) state->readBuf; pageHeaderSize = XLogPageHeaderSize(header); - /* make sure we have enough data for the page header */ - readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); - if (readLen < 0) - goto err; + /* we should have read the page header */ + Assert(state->readLen >= pageHeaderSize); /* skip over potential continuation data */ if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 88a1bfd939..421040f391 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -706,8 +706,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum, void XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength) { - const XLogRecPtr lastReadPage = (state->seg.ws_segno * - state->segcxt.ws_segsize + state->segoff); + const XLogRecPtr lastReadPage = state->seg.ws_segno * + state->segcxt.ws_segsize + state->readLen; Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0); Assert(wantLength <= XLOG_BLCKSZ); @@ -844,7 +844,7 @@ wal_segment_close(XLogReaderState *state) * exists for normal backends, so we have to do a check/sleep/repeat style of * loop for now. */ -int +bool read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page) { @@ -946,7 +946,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, else if (targetPagePtr + reqLen > read_upto) { /* not enough data there */ - return -1; + state->readLen = -1; + return false; } else { @@ -964,7 +965,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, WALReadRaiseError(&errinfo); /* number of valid bytes in the buffer */ - return count; + state->readLen = count; + return true; } /* diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index b811a5c0ef..ddba340653 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -802,7 +802,7 @@ StartReplication(StartReplicationCmd *cmd) * which has to do a plain sleep/busy loop, because the walsender's latch gets * set every time WAL is flushed. */ -static int +static bool logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page) { @@ -822,7 +822,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req /* fail if not (implies we are going to shut down) */ if (flushptr < targetPagePtr + reqLen) - return -1; + { + state->readLen = -1; + return false; + } if (targetPagePtr + XLOG_BLCKSZ <= flushptr) count = XLOG_BLCKSZ; /* more than one block available */ @@ -850,7 +853,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize); CheckXLogRemoved(segno, state->seg.ws_tli); - return count; + state->readLen = count; + return true; } /* diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 59ebac7d6a..cf119848b0 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate int tliIndex; } XLogPageReadPrivate; -static int SimpleXLogPageRead(XLogReaderState *xlogreader, +static bool SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf); @@ -246,7 +246,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, } /* XLogReader callback function, to read a WAL page */ -static int +static bool SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf) { @@ -306,7 +306,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (private->restoreCommand == NULL) { pg_log_error("could not open file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return false; } /* @@ -319,7 +320,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private->restoreCommand); if (xlogreadfd < 0) - return -1; + { + xlogreader->readLen = -1; + return false; + } else pg_log_debug("using file \"%s\" restored from archive", xlogfpath); @@ -335,7 +339,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) { pg_log_error("could not seek in file \"%s\": %m", xlogfpath); - return -1; + xlogreader->readLen = -1; + return false; } @@ -348,13 +353,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, pg_log_error("could not read file \"%s\": read %d of %zu", xlogfpath, r, (Size) XLOG_BLCKSZ); - return -1; + xlogreader->readLen = -1; + return false; } Assert(targetSegNo == xlogreadsegno); xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli; - return XLOG_BLCKSZ; + xlogreader->readLen = XLOG_BLCKSZ; + return true; } /* diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 1e3894b9c4..833a64210b 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -332,7 +332,7 @@ WALDumpCloseSegment(XLogReaderState *state) } /* pg_waldump's XLogReaderRoutine->page_read callback */ -static int +static bool WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr, char *readBuff) { @@ -349,7 +349,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, else { private->endptr_reached = true; - return -1; + state->readLen = -1; + return false; } } @@ -374,7 +375,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, errinfo.wre_req); } - return count; + state->readLen = count; + return true; } /* diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index de6fd791fe..f3cf4f2f49 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -57,12 +57,12 @@ typedef struct WALSegmentContext typedef struct XLogReaderState XLogReaderState; -/* Function type definitions for various xlogreader interactions */ -typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, - XLogRecPtr targetRecPtr, - char *readBuf); +/* Function type definition for the read_page callback */ +typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader, + XLogRecPtr targetPagePtr, + int reqLen, + XLogRecPtr targetRecPtr, + char *readBuf); typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p); @@ -185,6 +185,18 @@ struct XLogReaderState /* Set when XLP_FIRST_IS_OVERWRITE_CONTRECORD is found */ XLogRecPtr overwrittenRecPtr; + /* ---------------------------------------- + * Communication with page reader + * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes. + * ---------------------------------------- + */ + /* variables to communicate with page reader */ + XLogRecPtr readPagePtr; /* page pointer to read */ + int32 readLen; /* bytes requested to reader, or actual bytes + * read by reader, which must be larger than + * the request, or -1 on error */ + char *readBuf; /* buffer to store data */ + bool page_verified; /* is the page on the buffer verified? */ /* ---------------------------------------- * Decoded representation of current record @@ -213,13 +225,6 @@ struct XLogReaderState * ---------------------------------------- */ - /* - * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least - * readLen bytes) - */ - char *readBuf; - uint32 readLen; - /* last read XLOG position for data currently in readBuf */ WALSegmentContext segcxt; WALOpenSegment seg; diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index a5cb3d322c..8669f7eeb3 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -89,7 +89,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern int read_local_xlog_page(XLogReaderState *state, +extern bool read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page); extern void wal_segment_open(XLogReaderState *state, -- 2.27.0 From 320e6d1cc8d65c455069fb32390039601839c667 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Thu, 30 Sep 2021 12:04:37 +0900 Subject: [PATCH v18 2/5] Move page-reader out of XLogReadRecord(). This is the second step of removing callbacks from the WAL decoder. XLogReadRecord() return XLREAD_NEED_DATA to indicate that the caller should supply new data, and the decoder works as a state machine. Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> Author: Heikki Linnakangas <hlinnaka@iki.fi> Reviewed-by: Antonin Houska <ah@cybertec.at> Reviewed-by: Alvaro Herrera <alvherre@2ndquadrant.com> Reviewed-by: Takashi Menjo <takashi.menjo@gmail.com> Reviewed-by: Thomas Munro <thomas.munro@gmail.com> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/twophase.c | 16 +- src/backend/access/transam/xlog.c | 60 +- src/backend/access/transam/xlogreader.c | 760 ++++++++++-------- src/backend/access/transam/xlogutils.c | 17 +- src/backend/replication/logical/logical.c | 29 +- .../replication/logical/logicalfuncs.c | 16 +- src/backend/replication/slotfuncs.c | 21 +- src/backend/replication/walsender.c | 35 +- src/bin/pg_rewind/parsexlog.c | 93 ++- src/bin/pg_waldump/pg_waldump.c | 39 +- src/include/access/xlogreader.h | 124 ++- src/include/access/xlogutils.h | 4 +- src/include/pg_config_manual.h | 2 +- src/include/replication/logical.h | 11 +- 14 files changed, 703 insertions(+), 524 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 2156de187c..9b42a935a7 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1330,11 +1330,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) char *errormsg; TimeLineID save_currtli = ThisTimeLineID; - xlogreader = XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.page_read = &read_local_xlog_page, - .segment_open = &wal_segment_open, - .segment_close = &wal_segment_close), - NULL); + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -1342,7 +1339,14 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len) errdetail("Failed while allocating a WAL reading processor."))); XLogBeginRead(xlogreader, lsn); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == XLREAD_NEED_DATA) + { + if (!read_local_xlog_page(xlogreader)) + { + XLogTerminateRead(xlogreader); + break; + } + } /* * Restore immediately the timeline where it was previously, as diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 1557ceb8c1..316ba256f6 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -836,13 +836,6 @@ static XLogSource currentSource = XLOG_FROM_ANY; static bool lastSourceFailed = false; static bool pendingWalRcvRestart = false; -typedef struct XLogPageReadPrivate -{ - int emode; - bool fetching_ckpt; /* are we fetching a checkpoint record? */ - bool randAccess; -} XLogPageReadPrivate; - /* * These variables track when we last obtained some WAL data to process, * and where we got it from. (XLogReceiptSource is initially the same as @@ -918,8 +911,8 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf); +static bool XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, bool fetching_ckpt, XLogRecPtr tliRecPtr); static void XLogShutdownWalRcv(void); @@ -1233,8 +1226,7 @@ XLogInsertRecord(XLogRecData *rdata, appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len); if (!debug_reader) - debug_reader = XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(), NULL); + debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL); if (!debug_reader) { @@ -4395,15 +4387,10 @@ CleanupBackupHistory(void) * record is available. */ static XLogRecord * -ReadRecord(XLogReaderState *xlogreader, int emode, - bool fetching_ckpt) +ReadRecord(XLogReaderState *xlogreader, int emode, bool fetching_ckpt) { XLogRecord *record; - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - - private->fetching_ckpt = fetching_ckpt; - private->emode = emode; - private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr); + bool randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr); /* This is the first attempt to read this page. */ lastSourceFailed = false; @@ -4411,8 +4398,18 @@ ReadRecord(XLogReaderState *xlogreader, int emode, for (;;) { char *errormsg; + XLogReadRecordResult result; + + while ((result = XLogReadRecord(xlogreader, &record, &errormsg)) + == XLREAD_NEED_DATA) + { + if (!XLogPageRead(xlogreader, fetching_ckpt, emode, randAccess)) + { + XLogTerminateRead(xlogreader); + break; + } + } - record = XLogReadRecord(xlogreader, &errormsg); ReadRecPtr = xlogreader->ReadRecPtr; EndRecPtr = xlogreader->EndRecPtr; if (record == NULL) @@ -6547,7 +6544,6 @@ StartupXLOG(void) bool backupFromStandby = false; DBState dbstate_at_startup; XLogReaderState *xlogreader; - XLogPageReadPrivate private; bool promoted = false; struct stat st; @@ -6706,13 +6702,9 @@ StartupXLOG(void) OwnLatch(&XLogCtl->recoveryWakeupLatch); /* Set up XLOG reader facility */ - MemSet(&private, 0, sizeof(XLogPageReadPrivate)); xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.page_read = &XLogPageRead, - .segment_open = NULL, - .segment_close = wal_segment_close), - &private); + XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); + if (!xlogreader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -12300,12 +12292,13 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf) +XLogPageRead(XLogReaderState *xlogreader, + bool fetching_ckpt, int emode, bool randAccess) { - XLogPageReadPrivate *private = - (XLogPageReadPrivate *) xlogreader->private_data; - int emode = private->emode; + char *readBuf = xlogreader->readBuf; + XLogRecPtr targetPagePtr = xlogreader->readPagePtr; + int reqLen = xlogreader->readLen; + XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -12348,8 +12341,8 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - private->randAccess, - private->fetching_ckpt, + randAccess, + fetching_ckpt, targetRecPtr)) { if (readFile >= 0) @@ -12469,6 +12462,7 @@ retry: goto next_record_is_invalid; } + Assert(xlogreader->readPagePtr == targetPagePtr); xlogreader->readLen = readLen; return true; diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 1d9976ecf4..4501a09245 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -43,7 +43,7 @@ static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, bool header_inclusive); static void XLogReaderInvalReadState(XLogReaderState *state); static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, - XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); + XLogRecPtr PrevRecPtr, XLogRecord *record); static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr); static void ResetDecoder(XLogReaderState *state); @@ -76,7 +76,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...) */ XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, - XLogReaderRoutine *routine, void *private_data) + WALSegmentCleanupCB cleanup_cb) { XLogReaderState *state; @@ -87,7 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, return NULL; /* initialize caller-provided support functions */ - state->routine = *routine; + state->cleanup_cb = cleanup_cb; state->max_block_id = -1; @@ -110,8 +110,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size, waldir); - /* system_identifier initialized to zeroes above */ - state->private_data = private_data; /* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */ state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1, MCXT_ALLOC_NO_OOM); @@ -143,8 +141,8 @@ XLogReaderFree(XLogReaderState *state) { int block_id; - if (state->seg.ws_file != -1) - state->routine.segment_close(state); + if (state->seg.ws_file >= 0) + state->cleanup_cb(state); for (block_id = 0; block_id <= XLR_MAX_BLOCK_ID; block_id++) { @@ -249,6 +247,7 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) /* Begin at the passed-in record pointer. */ state->EndRecPtr = RecPtr; state->ReadRecPtr = InvalidXLogRecPtr; + state->readRecordState = XLREAD_NEXT_RECORD; } /* @@ -257,12 +256,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) * XLogBeginRead() or XLogFindNextRecord() must be called before the first call * to XLogReadRecord(). * - * If the page_read callback fails to read the requested data, NULL is - * returned. The callback is expected to have reported the error; errormsg - * is set to NULL. + * This function may return XLREAD_NEED_DATA several times before returning a + * result record. The caller shall read in some new data then call this + * function again with the same parameters. * - * If the reading fails for some other reason, NULL is also returned, and - * *errormsg is set to a string with details of the failure. + * When a record is successfully read, returns XLREAD_SUCCESS with result + * record being stored in *record. Otherwise *record is NULL. * * Returns XLREAD_NEED_DATA if more data is needed to finish reading the * current record. In that case, state->readPagePtr and state->readLen inform @@ -307,329 +306,458 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) * state. This behavior allows to continue reading a reacord switching among * different souces, while streaming replication. */ -XLogRecord * -XLogReadRecord(XLogReaderState *state, char **errormsg) +XLogReadRecordResult +XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg) { - XLogRecPtr RecPtr; - XLogRecord *record; - XLogRecPtr targetPagePtr; - bool randAccess; - uint32 len, - total_len; - uint32 targetRecOff; - uint32 pageHeaderSize; - bool assembled; - bool gotheader; + XLogRecord *prec; - /* - * randAccess indicates whether to verify the previous-record pointer of - * the record we're reading. We only do this if we're reading - * sequentially, which is what we initially assume. - */ - randAccess = false; + *record = NULL; /* reset error state */ *errormsg = NULL; state->errormsg_buf[0] = '\0'; - ResetDecoder(state); state->abortedRecPtr = InvalidXLogRecPtr; state->missingContrecPtr = InvalidXLogRecPtr; - RecPtr = state->EndRecPtr; - - if (state->ReadRecPtr != InvalidXLogRecPtr) - { - /* read the record after the one we just read */ - - /* - * EndRecPtr is pointing to end+1 of the previous WAL record. If - * we're at a page boundary, no more records can fit on the current - * page. We must skip over the page header, but we can't do that until - * we've read in the page, since the header size is variable. - */ - } - else - { - /* - * Caller supplied a position to start at. - * - * In this case, EndRecPtr should already be pointing to a valid - * record starting position. - */ - Assert(XRecOffIsValid(RecPtr)); - randAccess = true; - } - restart: - state->currRecPtr = RecPtr; - assembled = false; - - targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ); - targetRecOff = RecPtr % XLOG_BLCKSZ; - - /* - * Read the page containing the record into state->readBuf. Request enough - * byte to cover the whole record header, or at least the part of it that - * fits on the same page. - */ - while (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), - targetRecOff != 0)) + switch (state->readRecordState) { - if (!state->routine.page_read(state, state->readPagePtr, state->readLen, - RecPtr, state->readBuf)) - break; - } - - if (!state->page_verified) - goto err; - - /* - * We have at least the page header, so we can examine it now. - */ - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); - if (targetRecOff == 0) - { - /* - * At page start, so skip over page header. - */ - RecPtr += pageHeaderSize; - targetRecOff = pageHeaderSize; - } - else if (targetRecOff < pageHeaderSize) - { - report_invalid_record(state, "invalid record offset at %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; - } + case XLREAD_NEXT_RECORD: + ResetDecoder(state); - if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) && - targetRecOff == pageHeaderSize) - { - report_invalid_record(state, "contrecord is requested by %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; - } - - /* XLogNeedData has verified the page header */ - Assert(pageHeaderSize <= state->readLen); - - /* - * Read the record length. - * - * NB: Even though we use an XLogRecord pointer here, the whole record - * header might not fit on this page. xl_tot_len is the first field of the - * struct, so it must be on this page (the records are MAXALIGNed), but we - * cannot access any other fields until we've verified that we got the - * whole header. - */ - record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); - total_len = record->xl_tot_len; - - /* - * If the whole record header is on this page, validate it immediately. - * Otherwise do just a basic sanity check on xl_tot_len, and validate the - * rest of the header after reading it from the next page. The xl_tot_len - * check is necessary here to ensure that we enter the "Need to reassemble - * record" code path below; otherwise we might fail to apply - * ValidXLogRecordHeader at all. - */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) - { - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record, - randAccess)) - goto err; - gotheader = true; - } - else - { - /* XXX: more validation should be done here */ - if (total_len < SizeOfXLogRecord) - { - report_invalid_record(state, - "invalid record length at %X/%X: wanted %u, got %u", - LSN_FORMAT_ARGS(RecPtr), - (uint32) SizeOfXLogRecord, total_len); - goto err; - } - gotheader = false; - } - - len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ; - if (total_len > len) - { - /* Need to reassemble record */ - char *contdata; - XLogPageHeader pageHeader; - char *buffer; - uint32 gotlen; - - assembled = true; - - /* - * Enlarge readRecordBuf as needed. - */ - if (total_len > state->readRecordBufSize && - !allocate_recordbuf(state, total_len)) - { - /* We treat this as a "bogus data" condition */ - report_invalid_record(state, "record length %u at %X/%X too long", - total_len, LSN_FORMAT_ARGS(RecPtr)); - goto err; - } - - /* Copy the first fragment of the record from the first page. */ - memcpy(state->readRecordBuf, - state->readBuf + RecPtr % XLOG_BLCKSZ, len); - buffer = state->readRecordBuf + len; - gotlen = len; - - do - { - int rest_len = total_len - gotlen; - - /* Calculate pointer to beginning of next page */ - targetPagePtr += XLOG_BLCKSZ; - - /* Wait for the next page to become available */ - while (XLogNeedData(state, targetPagePtr, - Min(rest_len, XLOG_BLCKSZ), - false)) + if (state->ReadRecPtr != InvalidXLogRecPtr) + { + /* read the record after the one we just read */ + + /* + * EndRecPtr is pointing to end+1 of the previous WAL record. + * If we're at a page boundary, no more records can fit on the + * current page. We must skip over the page header, but we + * can't do that until we've read in the page, since the + * header size is variable. + */ + state->PrevRecPtr = state->ReadRecPtr; + state->ReadRecPtr = state->EndRecPtr; + } + else { - if (!state->routine.page_read(state, state->readPagePtr, - state->readLen, - state->ReadRecPtr, - state->readBuf)) + /* + * Caller supplied a position to start at. + * + * In this case, EndRecPtr should already be pointing to a + * valid record starting position. + */ + Assert(XRecOffIsValid(state->EndRecPtr)); + state->ReadRecPtr = state->EndRecPtr; + + /* + * We cannot verify the previous-record pointer when we're + * seeking to a particular record. Reset PrevRecPtr so that we + * won't try doing that. + */ + state->PrevRecPtr = InvalidXLogRecPtr; + state->EndRecPtr = InvalidXLogRecPtr; /* to be tidy */ + } + + state->record_verified = false; + state->readRecordState = XLREAD_TOT_LEN; + /* fall through */ + + case XLREAD_TOT_LEN: + { + uint32 total_len; + uint32 pageHeaderSize; + XLogRecPtr targetPagePtr; + uint32 targetRecOff; + XLogPageHeader pageHeader; + + targetPagePtr = + state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); + targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + + /* + * Check if we have enough data. For the first record in the + * page, the requesting length doesn't contain page header. + */ + if (XLogNeedData(state, targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ), + targetRecOff != 0)) + return XLREAD_NEED_DATA; + + /* error out if caller supplied bogus page */ + if (!state->page_verified) + goto err; + + /* examine page header now. */ + pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + if (targetRecOff == 0) + { + /* At page start, so skip over page header. */ + state->ReadRecPtr += pageHeaderSize; + targetRecOff = pageHeaderSize; + } + else if (targetRecOff < pageHeaderSize) + { + report_invalid_record(state, "invalid record offset at %X/%X", + LSN_FORMAT_ARGS(state->ReadRecPtr)); + goto err; + } + + pageHeader = (XLogPageHeader) state->readBuf; + if ((pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD) && + targetRecOff == pageHeaderSize) + { + report_invalid_record(state, "contrecord is requested by %X/%X", + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr); + goto err; + } + + /* XLogNeedData has verified the page header */ + Assert(pageHeaderSize <= state->readLen); + + /* + * Read the record length. + * + * NB: Even though we use an XLogRecord pointer here, the + * whole record header might not fit on this page. xl_tot_len + * is the first field of the struct, so it must be on this + * page (the records are MAXALIGNed), but we cannot access any + * other fields until we've verified that we got the whole + * header. + */ + prec = (XLogRecord *) (state->readBuf + + state->ReadRecPtr % XLOG_BLCKSZ); + total_len = prec->xl_tot_len; + + /* + * If the whole record header is on this page, validate it + * immediately. Otherwise do just a basic sanity check on + * xl_tot_len, and validate the rest of the header after + * reading it from the next page. The xl_tot_len check is + * necessary here to ensure that we enter the + * XLREAD_CONTINUATION state below; otherwise we might fail to + * apply ValidXLogRecordHeader at all. + */ + if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + { + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, prec)) + goto err; + + state->record_verified = true; + } + else + { + /* XXX: more validation should be done here */ + if (total_len < SizeOfXLogRecord) + { + report_invalid_record(state, + "invalid record length at %X/%X: wanted %u, got %u", + LSN_FORMAT_ARGS(state->ReadRecPtr), + (uint32) SizeOfXLogRecord, total_len); + goto err; + } + } + + /* + * Wait for the rest of the record, or the part of the record + * that fit on the first page if crossed a page boundary, to + * become available. + */ + state->recordGotLen = 0; + state->recordRemainLen = total_len; + state->readRecordState = XLREAD_FIRST_FRAGMENT; + } + /* fall through */ + + case XLREAD_FIRST_FRAGMENT: + { + uint32 total_len = state->recordRemainLen; + uint32 request_len; + uint32 record_len; + XLogRecPtr targetPagePtr; + uint32 targetRecOff; + + /* + * Wait for the rest of the record on the first page to become + * available + */ + targetPagePtr = + state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ); + targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ; + + request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ); + record_len = request_len - targetRecOff; + + /* ReadRecPtr contains page header */ + Assert(targetRecOff != 0); + if (XLogNeedData(state, targetPagePtr, request_len, true)) + return XLREAD_NEED_DATA; + + /* error out if caller supplied bogus page */ + if (!state->page_verified) + goto err; + + prec = (XLogRecord *) (state->readBuf + targetRecOff); + + /* validate record header if not yet */ + if (!state->record_verified && record_len >= SizeOfXLogRecord) + { + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, prec)) + goto err; + + state->record_verified = true; + } + + + if (total_len == record_len) + { + /* Record does not cross a page boundary */ + Assert(state->record_verified); + + if (!ValidXLogRecord(state, prec, state->ReadRecPtr)) + goto err; + + state->record_verified = true; /* to be tidy */ + + /* We already checked the header earlier */ + state->EndRecPtr = state->ReadRecPtr + MAXALIGN(record_len); + + *record = prec; + state->readRecordState = XLREAD_NEXT_RECORD; break; - } + } - if (!state->page_verified) - goto err; + /* + * The record continues on the next page. Need to reassemble + * record + */ + Assert(total_len > record_len); - Assert(SizeOfXLogShortPHD <= state->readLen); + /* Enlarge readRecordBuf as needed. */ + if (total_len > state->readRecordBufSize && + !allocate_recordbuf(state, total_len)) + { + /* We treat this as a "bogus data" condition */ + report_invalid_record(state, + "record length %u at %X/%X too long", + total_len, + LSN_FORMAT_ARGS(state->ReadRecPtr)); + goto err; + } - pageHeader = (XLogPageHeader) state->readBuf; + /* Copy the first fragment of the record from the first page. */ + memcpy(state->readRecordBuf, state->readBuf + targetRecOff, + record_len); + state->recordGotLen += record_len; + state->recordRemainLen -= record_len; - /* - * If we were expecting a continuation record and got an - * "overwrite contrecord" flag, that means the continuation record - * was overwritten with a different record. Restart the read by - * assuming the address to read is the location where we found - * this flag; but keep track of the LSN of the record we were - * reading, for later verification. - */ - if (pageHeader->xlp_info & XLP_FIRST_IS_OVERWRITE_CONTRECORD) - { - state->overwrittenRecPtr = state->currRecPtr; - ResetDecoder(state); - RecPtr = targetPagePtr; - goto restart; - } + /* Calculate pointer to beginning of next page */ + state->recordContRecPtr = state->ReadRecPtr + record_len; + Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0); - /* Check that the continuation on next page looks valid */ - if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) - { - report_invalid_record(state, - "there is no contrecord flag at %X/%X", - LSN_FORMAT_ARGS(RecPtr)); - goto err; + state->readRecordState = XLREAD_CONTINUATION; } + /* fall through */ - /* - * Cross-check that xlp_rem_len agrees with how much of the record - * we expect there to be left. - */ - if (pageHeader->xlp_rem_len == 0 || - total_len != (pageHeader->xlp_rem_len + gotlen)) + case XLREAD_CONTINUATION: { - report_invalid_record(state, - "invalid contrecord length %u (expected %lld) at %X/%X", - pageHeader->xlp_rem_len, - ((long long) total_len) - gotlen, - LSN_FORMAT_ARGS(RecPtr)); - goto err; - } + XLogPageHeader pageHeader; + uint32 pageHeaderSize; + XLogRecPtr targetPagePtr; - /* Append the continuation from this page to the buffer */ - pageHeaderSize = XLogPageHeaderSize(pageHeader); + /* + * we enter this state only if we haven't read the whole + * record. + */ + Assert(state->recordRemainLen > 0); - Assert(pageHeaderSize <= state->readLen); + while (state->recordRemainLen > 0) + { + char *contdata; + uint32 request_len; + uint32 record_len; - contdata = (char *) state->readBuf + pageHeaderSize; - len = XLOG_BLCKSZ - pageHeaderSize; - if (pageHeader->xlp_rem_len < len) - len = pageHeader->xlp_rem_len; + /* Wait for the next page to become available */ + targetPagePtr = state->recordContRecPtr; - Assert(pageHeaderSize + len <= state->readLen); - memcpy(buffer, (char *) contdata, len); - buffer += len; - gotlen += len; + /* this request contains page header */ + Assert(targetPagePtr != 0); + if (XLogNeedData(state, targetPagePtr, + Min(state->recordRemainLen, XLOG_BLCKSZ), + false)) + return XLREAD_NEED_DATA; - /* If we just reassembled the record header, validate it. */ - if (!gotheader) - { - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, - record, randAccess)) + if (!state->page_verified) + goto err; + + Assert(SizeOfXLogShortPHD <= state->readLen); + + /* Check that the continuation on next page looks valid */ + pageHeader = (XLogPageHeader) state->readBuf; + + /* + * If we were expecting a continuation record and got an + * "overwrite contrecord" flag, that means the continuation + * record was overwritten with a different record. Restart + * the read by assuming the address to read is the location + * where we found this flag; but keep track of the LSN of + * the record we were reading, for later verification. + */ + if (pageHeader->xlp_info & + XLP_FIRST_IS_OVERWRITE_CONTRECORD) + { + state->overwrittenRecPtr = state->ReadRecPtr; + state->EndRecPtr = targetPagePtr; + state->readRecordState = XLREAD_NEXT_RECORD; + + /* + * ReadRecPtr is the PrevRecPtr of the next + * record. Keep the LSN at the retry. + */ + state->ReadRecPtr = state->PrevRecPtr; + + goto restart; + } + + if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) + { + report_invalid_record( + state, + "there is no contrecord flag at %X/%X reading %X/%X", + (uint32) (state->recordContRecPtr >> 32), + (uint32) state->recordContRecPtr, + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr); + goto err; + } + + /* + * Cross-check that xlp_rem_len agrees with how much of + * the record we expect there to be left. + */ + if (pageHeader->xlp_rem_len == 0 || + pageHeader->xlp_rem_len != state->recordRemainLen) + { + report_invalid_record( + state, + "invalid contrecord length %u at %X/%X reading %X/%X, expected %u", + pageHeader->xlp_rem_len, + (uint32) (state->recordContRecPtr >> 32), + (uint32) state->recordContRecPtr, + (uint32) (state->ReadRecPtr >> 32), + (uint32) state->ReadRecPtr, + state->recordRemainLen); + goto err; + } + + /* Append the continuation from this page to the buffer */ + pageHeaderSize = XLogPageHeaderSize(pageHeader); + + /* + * XLogNeedData should have ensured that the whole page + * header was read + */ + Assert(state->readLen >= pageHeaderSize); + + contdata = (char *) state->readBuf + pageHeaderSize; + record_len = XLOG_BLCKSZ - pageHeaderSize; + if (pageHeader->xlp_rem_len < record_len) + record_len = pageHeader->xlp_rem_len; + + request_len = record_len + pageHeaderSize; + + /* + * XLogNeedData should have ensured all needed data was + * read + */ + Assert(state->readLen >= request_len); + + memcpy(state->readRecordBuf + state->recordGotLen, + (char *) contdata, record_len); + state->recordGotLen += record_len; + state->recordRemainLen -= record_len; + + /* If we just reassembled the record header, validate it. */ + if (!state->record_verified) + { + Assert(state->recordGotLen >= SizeOfXLogRecord); + if (!ValidXLogRecordHeader(state, state->ReadRecPtr, + state->PrevRecPtr, + (XLogRecord *) state->readRecordBuf)) + goto err; + + state->record_verified = true; + } + + /* + * Calculate pointer to beginning of next page, and + * continue + */ + state->recordContRecPtr += XLOG_BLCKSZ; + } + + /* targetPagePtr is pointing the last-read page here */ + prec = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecord(state, prec, state->ReadRecPtr)) goto err; - gotheader = true; - } - } while (gotlen < total_len); - - Assert(gotheader); - - record = (XLogRecord *) state->readRecordBuf; - if (!ValidXLogRecord(state, record, RecPtr)) - goto err; - - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); - state->ReadRecPtr = RecPtr; - state->EndRecPtr = targetPagePtr + pageHeaderSize - + MAXALIGN(pageHeader->xlp_rem_len); - } - else - { - /* Wait for the record data to become available */ - while (XLogNeedData(state, targetPagePtr, - Min(targetRecOff + total_len, XLOG_BLCKSZ), true)) - { - if (!state->routine.page_read(state, state->readPagePtr, - state->readLen, - state->ReadRecPtr, state->readBuf)) + + pageHeaderSize = + XLogPageHeaderSize((XLogPageHeader) state->readBuf); + state->EndRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(pageHeader->xlp_rem_len); + + *record = prec; + state->readRecordState = XLREAD_NEXT_RECORD; break; - } - - if (!state->page_verified) - goto err; - - /* Record does not cross a page boundary */ - if (!ValidXLogRecord(state, record, RecPtr)) - goto err; - - state->EndRecPtr = RecPtr + MAXALIGN(total_len); - - state->ReadRecPtr = RecPtr; + } } /* * Special processing if it's an XLOG SWITCH record */ - if (record->xl_rmid == RM_XLOG_ID && - (record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) + if ((*record)->xl_rmid == RM_XLOG_ID && + ((*record)->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH) { /* Pretend it extends to end of segment */ state->EndRecPtr += state->segcxt.ws_segsize - 1; state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize); } - if (DecodeXLogRecord(state, record, errormsg)) - return record; - else - return NULL; + Assert(!*record || state->readLen >= 0); + if (DecodeXLogRecord(state, *record, errormsg)) + return XLREAD_SUCCESS; + + *record = NULL; + return XLREAD_FAIL; err: - if (assembled) + Assert(state->readRecordState != XLREAD_NEXT_RECORD); + XLogTerminateRead(state); + + if (state->errormsg_buf[0] != '\0') + *errormsg = state->errormsg_buf; + + *record = NULL; + + return XLREAD_FAIL; +} + +/* + * Terminate read WAL. + * + * When the caller failed to read the data requested from XLogReadRecord, it is + * supposed to call this function to set the correct reader state to reflect + * the failure. + */ +void +XLogTerminateRead(XLogReaderState *state) +{ + if (state->readRecordState == XLREAD_CONTINUATION) { /* * We get here when a record that spans multiple pages needs to be @@ -640,20 +768,15 @@ err: * in turn signal downstream WAL consumers that the broken WAL record * is to be ignored. */ - state->abortedRecPtr = RecPtr; - state->missingContrecPtr = targetPagePtr; + state->abortedRecPtr = state->ReadRecPtr; + state->missingContrecPtr = state->recordContRecPtr; } /* - * Invalidate the read state. We might read from a different source after + * Invalidate the read page. We might read from a different source after * failure. */ XLogReaderInvalReadState(state); - - if (state->errormsg_buf[0] != '\0') - *errormsg = state->errormsg_buf; - - return NULL; } /* @@ -805,11 +928,12 @@ XLogReaderInvalReadState(XLogReaderState *state) * * This is just a convenience subroutine to avoid duplicated code in * XLogReadRecord. It's not intended for use from anywhere else. + * + * If PrevRecPtr is valid, the xl_prev is is cross-checked with it. */ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, - XLogRecPtr PrevRecPtr, XLogRecord *record, - bool randAccess) + XLogRecPtr PrevRecPtr, XLogRecord *record) { if (record->xl_tot_len < SizeOfXLogRecord) { @@ -826,7 +950,7 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, record->xl_rmid, LSN_FORMAT_ARGS(RecPtr)); return false; } - if (randAccess) + if (PrevRecPtr == InvalidXLogRecPtr) { /* * We can't exactly verify the prev-link, but surely it should be less @@ -1056,11 +1180,14 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * XLogReadRecord() will read the next valid record. */ XLogRecPtr -XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) +XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private) { XLogRecPtr tmpRecPtr; XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; + XLogRecord *record; + XLogReadRecordResult result; char *errormsg; Assert(!XLogRecPtrIsInvalid(RecPtr)); @@ -1093,9 +1220,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) while (XLogNeedData(state, targetPagePtr, targetRecOff, targetRecOff != 0)) { - if (!state->routine.page_read(state, state->readPagePtr, - state->readLen, - state->ReadRecPtr, state->readBuf)) + if (!read_page(state, private)) break; } @@ -1147,8 +1272,16 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) * or we just jumped over the remaining data of a continuation. */ XLogBeginRead(state, tmpRecPtr); - while (XLogReadRecord(state, &errormsg) != NULL) + while ((result = XLogReadRecord(state, &record, &errormsg)) != + XLREAD_FAIL) { + if (result == XLREAD_NEED_DATA) + { + if (!read_page(state, private)) + goto err; + continue; + } + /* past the record we've found, break out */ if (RecPtr <= state->ReadRecPtr) { @@ -1168,9 +1301,9 @@ err: #endif /* FRONTEND */ /* - * Helper function to ease writing of XLogRoutine->page_read callbacks. - * If this function is used, caller must supply a segment_open callback in - * 'state', as that is used here. + * Helper function to ease writing of page_read callback. + * If this function is used, caller must supply a segment_open callback and + * segment_close callback as that is used here. * * Read 'count' bytes into 'buf', starting at location 'startptr', from WAL * fetched from timeline 'tli'. @@ -1183,6 +1316,7 @@ err: */ bool WALRead(XLogReaderState *state, + WALSegmentOpenCB segopenfn, WALSegmentCloseCB segclosefn, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo) { @@ -1214,10 +1348,10 @@ WALRead(XLogReaderState *state, XLogSegNo nextSegNo; if (state->seg.ws_file >= 0) - state->routine.segment_close(state); + segclosefn(state); XLByteToSeg(recptr, nextSegNo, state->segcxt.ws_segsize); - state->routine.segment_open(state, nextSegNo, &tli); + segopenfn(state, nextSegNo, &tli); /* This shouldn't happen -- indicates a bug in segment_open */ Assert(state->seg.ws_file >= 0); diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 421040f391..fc09a72b8b 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -706,8 +706,7 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum, void XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength) { - const XLogRecPtr lastReadPage = state->seg.ws_segno * - state->segcxt.ws_segsize + state->readLen; + const XLogRecPtr lastReadPage = state->readPagePtr; Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0); Assert(wantLength <= XLOG_BLCKSZ); @@ -722,7 +721,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa * current TLI has since become historical. */ if (lastReadPage == wantPage && - state->readLen != 0 && + state->page_verified && lastReadPage + state->readLen >= wantPage + Min(wantLength, XLOG_BLCKSZ - 1)) return; @@ -808,6 +807,7 @@ wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, char path[MAXPGPATH]; XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize); + elog(LOG, "HOGE: %lu, %d => %s", nextSegNo, tli, path); state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY); if (state->seg.ws_file >= 0) return; @@ -845,9 +845,11 @@ wal_segment_close(XLogReaderState *state) * loop for now. */ bool -read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *cur_page) +read_local_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + char *cur_page = state->readBuf; XLogRecPtr read_upto, loc; TimeLineID tli; @@ -960,11 +962,12 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, * as 'count', read the whole page anyway. It's guaranteed to be * zero-padded up to the page boundary if it's incomplete. */ - if (!WALRead(state, cur_page, targetPagePtr, XLOG_BLCKSZ, tli, - &errinfo)) + if (!WALRead(state, wal_segment_open, wal_segment_close, + cur_page, targetPagePtr, XLOG_BLCKSZ, tli, &errinfo)) WALReadRaiseError(&errinfo); /* number of valid bytes in the buffer */ + state->readPagePtr = targetPagePtr; state->readLen = count; return true; } diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index aae0ae5b8a..2a88aa063a 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -148,7 +148,8 @@ StartupDecodingContext(List *output_plugin_options, TransactionId xmin_horizon, bool need_full_snapshot, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -198,11 +199,12 @@ StartupDecodingContext(List *output_plugin_options, ctx->slot = slot; - ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, xl_routine, ctx); + ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb); if (!ctx->reader) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + ctx->page_read = page_read; ctx->reorder = ReorderBufferAllocate(); ctx->snapshot_builder = @@ -319,7 +321,8 @@ CreateInitDecodingContext(const char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -422,7 +425,7 @@ CreateInitDecodingContext(const char *plugin, ctx = StartupDecodingContext(NIL, restart_lsn, xmin_horizon, need_full_snapshot, false, - xl_routine, prepare_write, do_write, + page_read, cleanup_cb, prepare_write, do_write, update_progress); /* call output plugin initialization callback */ @@ -478,7 +481,8 @@ LogicalDecodingContext * CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress) @@ -534,8 +538,8 @@ CreateDecodingContext(XLogRecPtr start_lsn, ctx = StartupDecodingContext(output_plugin_options, start_lsn, InvalidTransactionId, false, - fast_forward, xl_routine, prepare_write, - do_write, update_progress); + fast_forward, page_read, cleanup_cb, + prepare_write, do_write, update_progress); /* call output plugin initialization callback */ old_context = MemoryContextSwitchTo(ctx->context); @@ -603,7 +607,16 @@ DecodingContextFindStartpoint(LogicalDecodingContext *ctx) char *err = NULL; /* the read_page callback waits for new WAL */ - record = XLogReadRecord(ctx->reader, &err); + while (XLogReadRecord(ctx->reader, &record, &err) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + { + XLogTerminateRead(ctx->reader); + break; + } + } + if (err) elog(ERROR, "%s", err); if (!record) diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c index e59939aad1..f68d08fdee 100644 --- a/src/backend/replication/logical/logicalfuncs.c +++ b/src/backend/replication/logical/logicalfuncs.c @@ -224,9 +224,8 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin ctx = CreateDecodingContext(InvalidXLogRecPtr, options, false, - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, LogicalOutputPrepareWrite, LogicalOutputWrite, NULL); @@ -275,7 +274,16 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin XLogRecord *record; char *errm = NULL; - record = XLogReadRecord(ctx->reader, &errm); + while (XLogReadRecord(ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + { + XLogTerminateRead(ctx->reader); + break; + } + } + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index 17df99c2ac..57426aa9d7 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -144,9 +144,8 @@ create_logical_replication_slot(char *name, char *plugin, ctx = CreateInitDecodingContext(plugin, NIL, false, /* just catalogs is OK */ restart_lsn, - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, NULL, NULL, NULL); /* @@ -503,9 +502,8 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) ctx = CreateDecodingContext(InvalidXLogRecPtr, NIL, true, /* fast_forward */ - XL_ROUTINE(.page_read = read_local_xlog_page, - .segment_open = wal_segment_open, - .segment_close = wal_segment_close), + read_local_xlog_page, + wal_segment_close, NULL, NULL, NULL); /* @@ -527,7 +525,16 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) * Read records. No changes are generated in fast_forward mode, * but snapbuilder/slot statuses are updated properly. */ - record = XLogReadRecord(ctx->reader, &errm); + while (XLogReadRecord(ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!ctx->page_read(ctx->reader)) + { + XLogTerminateRead(ctx->reader); + break; + } + } + if (errm) elog(ERROR, "%s", errm); diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index ddba340653..78d805ab80 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -575,10 +575,7 @@ StartReplication(StartReplicationCmd *cmd) /* create xlogreader for physical replication */ xlogreader = - XLogReaderAllocate(wal_segment_size, NULL, - XL_ROUTINE(.segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), - NULL); + XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close); if (!xlogreader) ereport(ERROR, @@ -803,9 +800,11 @@ StartReplication(StartReplicationCmd *cmd) * set every time WAL is flushed. */ static bool -logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page) +logical_read_xlog_page(XLogReaderState *state) { + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + char *cur_page = state->readBuf; XLogRecPtr flushptr; int count; WALReadError errinfo; @@ -833,7 +832,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req count = flushptr - targetPagePtr; /* part of the page available */ /* now actually read the data, we know it's there */ - if (!WALRead(state, + if (!WALRead(state, WalSndSegmentOpen, wal_segment_close, cur_page, targetPagePtr, XLOG_BLCKSZ, @@ -1023,9 +1022,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) ctx = CreateInitDecodingContext(cmd->plugin, NIL, need_full_snapshot, InvalidXLogRecPtr, - XL_ROUTINE(.page_read = logical_read_xlog_page, - .segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), + logical_read_xlog_page, + wal_segment_close, WalSndPrepareWrite, WalSndWriteData, WalSndUpdateProgress); @@ -1183,9 +1181,8 @@ StartLogicalReplication(StartReplicationCmd *cmd) */ logical_decoding_ctx = CreateDecodingContext(cmd->startpoint, cmd->options, false, - XL_ROUTINE(.page_read = logical_read_xlog_page, - .segment_open = WalSndSegmentOpen, - .segment_close = wal_segment_close), + logical_read_xlog_page, + wal_segment_close, WalSndPrepareWrite, WalSndWriteData, WalSndUpdateProgress); xlogreader = logical_decoding_ctx->reader; @@ -2778,7 +2775,7 @@ XLogSendPhysical(void) enlargeStringInfo(&output_message, nbytes); retry: - if (!WALRead(xlogreader, + if (!WALRead(xlogreader, WalSndSegmentOpen, wal_segment_close, &output_message.data[output_message.len], startptr, nbytes, @@ -2876,7 +2873,15 @@ XLogSendLogical(void) */ WalSndCaughtUp = false; - record = XLogReadRecord(logical_decoding_ctx->reader, &errm); + while (XLogReadRecord(logical_decoding_ctx->reader, &record, &errm) == + XLREAD_NEED_DATA) + { + if (!logical_decoding_ctx->page_read(logical_decoding_ctx->reader)) + { + XLogTerminateRead(logical_decoding_ctx->reader); + break; + } + } /* xlog record was invalid */ if (errm != NULL) diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index cf119848b0..da723e5340 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -41,15 +41,9 @@ static int xlogreadfd = -1; static XLogSegNo xlogreadsegno = -1; static char xlogfpath[MAXPGPATH]; -typedef struct XLogPageReadPrivate -{ - const char *restoreCommand; - int tliIndex; -} XLogPageReadPrivate; - -static bool SimpleXLogPageRead(XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf); +static bool SimpleXLogPageRead(XLogReaderState *xlogreader, + const char *datadir, int *tliIndex, + const char *restoreCommand); /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline @@ -66,20 +60,25 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); + if (xlogreader == NULL) pg_fatal("out of memory"); XLogBeginRead(xlogreader, startpoint); do { - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, + &tliIndex, restoreCommand)) + { + XLogTerminateRead(xlogreader); + break; + } + } if (record == NULL) { @@ -123,19 +122,22 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; XLogRecPtr endptr; - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); XLogBeginRead(xlogreader, ptr); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, &tliIndex, restoreCommand)) + { + XLogTerminateRead(xlogreader); + break; + } + } if (record == NULL) { if (errormsg) @@ -170,7 +172,6 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr searchptr; XLogReaderState *xlogreader; char *errormsg; - XLogPageReadPrivate private; /* * The given fork pointer points to the end of the last common record, @@ -186,11 +187,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, forkptr += SizeOfXLogShortPHD; } - private.tliIndex = tliIndex; - private.restoreCommand = restoreCommand; - xlogreader = XLogReaderAllocate(WalSegSz, datadir, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - &private); + xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL); if (xlogreader == NULL) pg_fatal("out of memory"); @@ -200,7 +197,16 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, uint8 info; XLogBeginRead(xlogreader, searchptr); - record = XLogReadRecord(xlogreader, &errormsg); + while (XLogReadRecord(xlogreader, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!SimpleXLogPageRead(xlogreader, datadir, + &tliIndex, restoreCommand)) + { + XLogTerminateRead(xlogreader); + break; + } + } if (record == NULL) { @@ -247,10 +253,11 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, /* XLogReader callback function, to read a WAL page */ static bool -SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf) +SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, + int *tliIndex, const char *restoreCommand) { - XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; + XLogRecPtr targetPagePtr = xlogreader->readPagePtr; + char *readBuf = xlogreader->readBuf; uint32 targetPageOff; XLogRecPtr targetSegEnd; XLogSegNo targetSegNo; @@ -283,14 +290,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, * be done both forward and backward, consider also switching timeline * accordingly. */ - while (private->tliIndex < targetNentries - 1 && - targetHistory[private->tliIndex].end < targetSegEnd) - private->tliIndex++; - while (private->tliIndex > 0 && - targetHistory[private->tliIndex].begin >= targetSegEnd) - private->tliIndex--; + while (*tliIndex < targetNentries - 1 && + targetHistory[*tliIndex].end < targetSegEnd) + (*tliIndex)++; + while (*tliIndex > 0 && + targetHistory[*tliIndex].begin >= targetSegEnd) + (*tliIndex)--; - XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, + XLogFileName(xlogfname, targetHistory[*tliIndex].tli, xlogreadsegno, WalSegSz); snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", @@ -303,7 +310,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* * If we have no restore_command to execute, then exit. */ - if (private->restoreCommand == NULL) + if (restoreCommand == NULL) { pg_log_error("could not open file \"%s\": %m", xlogfpath); xlogreader->readLen = -1; @@ -317,7 +324,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir, xlogfname, WalSegSz, - private->restoreCommand); + restoreCommand); if (xlogreadfd < 0) { @@ -359,7 +366,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, Assert(targetSegNo == xlogreadsegno); - xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli; + xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli; xlogreader->readLen = XLOG_BLCKSZ; return true; } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 833a64210b..80182621f8 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -331,12 +331,17 @@ WALDumpCloseSegment(XLogReaderState *state) state->seg.ws_file = -1; } -/* pg_waldump's XLogReaderRoutine->page_read callback */ +/* + * pg_waldump's WAL page rader, also used as page_read callback for + * XLogFindNextRecord + */ static bool -WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetPtr, char *readBuff) +WALDumpReadPage(XLogReaderState *state, void *priv) { - XLogDumpPrivate *private = state->private_data; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + char *readBuff = state->readBuf; + XLogDumpPrivate *private = (XLogDumpPrivate *) priv; int count = XLOG_BLCKSZ; WALReadError errinfo; @@ -354,8 +359,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, } } - if (!WALRead(state, readBuff, targetPagePtr, count, private->timeline, - &errinfo)) + if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment, + readBuff, targetPagePtr, count, private->timeline, &errinfo)) { WALOpenSegment *seg = &errinfo.wre_seg; char fname[MAXPGPATH]; @@ -375,6 +380,7 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, errinfo.wre_req); } + Assert(count >= state->readLen); state->readLen = count; return true; } @@ -1057,16 +1063,14 @@ main(int argc, char **argv) /* we have everything we need, start reading */ xlogreader_state = - XLogReaderAllocate(WalSegSz, waldir, - XL_ROUTINE(.page_read = WALDumpReadPage, - .segment_open = WALDumpOpenSegment, - .segment_close = WALDumpCloseSegment), - &private); + XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment); + if (!xlogreader_state) fatal_error("out of memory"); /* first find a valid recptr to start from */ - first_record = XLogFindNextRecord(xlogreader_state, private.startptr); + first_record = XLogFindNextRecord(xlogreader_state, private.startptr, + &WALDumpReadPage, (void*) &private); if (first_record == InvalidXLogRecPtr) fatal_error("could not find a valid record after %X/%X", @@ -1089,7 +1093,16 @@ main(int argc, char **argv) for (;;) { /* try to read the next record */ - record = XLogReadRecord(xlogreader_state, &errormsg); + while (XLogReadRecord(xlogreader_state, &record, &errormsg) == + XLREAD_NEED_DATA) + { + if (!WALDumpReadPage(xlogreader_state, (void *) &private)) + { + XLogTerminateRead(xlogreader_state); + break; + } + } + if (!record) { if (!config.follow || private.endptr_reached) diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index f3cf4f2f49..ff1aca719b 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -57,64 +57,15 @@ typedef struct WALSegmentContext typedef struct XLogReaderState XLogReaderState; -/* Function type definition for the read_page callback */ -typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, - XLogRecPtr targetRecPtr, - char *readBuf); +/* Function type definition for the segment cleanup callback */ +typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader); + +/* Function type definition for the open/close callbacks for WALRead() */ typedef void (*WALSegmentOpenCB) (XLogReaderState *xlogreader, XLogSegNo nextSegNo, TimeLineID *tli_p); typedef void (*WALSegmentCloseCB) (XLogReaderState *xlogreader); -typedef struct XLogReaderRoutine -{ - /* - * Data input callback - * - * This callback shall read at least reqLen valid bytes of the xlog page - * starting at targetPagePtr, and store them in readBuf. The callback - * shall return the number of bytes read (never more than XLOG_BLCKSZ), or - * -1 on failure. The callback shall sleep, if necessary, to wait for the - * requested bytes to become available. The callback will not be invoked - * again for the same page unless more than the returned number of bytes - * are needed. - * - * targetRecPtr is the position of the WAL record we're reading. Usually - * it is equal to targetPagePtr + reqLen, but sometimes xlogreader needs - * to read and verify the page or segment header, before it reads the - * actual WAL record it's interested in. In that case, targetRecPtr can - * be used to determine which timeline to read the page from. - * - * The callback shall set ->seg.ws_tli to the TLI of the file the page was - * read from. - */ - XLogPageReadCB page_read; - - /* - * Callback to open the specified WAL segment for reading. ->seg.ws_file - * shall be set to the file descriptor of the opened segment. In case of - * failure, an error shall be raised by the callback and it shall not - * return. - * - * "nextSegNo" is the number of the segment to be opened. - * - * "tli_p" is an input/output argument. WALRead() uses it to pass the - * timeline in which the new segment should be found, but the callback can - * use it to return the TLI that it actually opened. - */ - WALSegmentOpenCB segment_open; - - /* - * WAL segment close callback. ->seg.ws_file shall be set to a negative - * number. - */ - WALSegmentCloseCB segment_close; -} XLogReaderRoutine; - -#define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__} - typedef struct { /* Is this block ref in use? */ @@ -144,12 +95,36 @@ typedef struct uint16 data_bufsz; } DecodedBkpBlock; +/* Return code from XLogReadRecord */ +typedef enum XLogReadRecordResult +{ + XLREAD_SUCCESS, /* record is successfully read */ + XLREAD_NEED_DATA, /* need more data. see XLogReadRecord. */ + XLREAD_FAIL /* failed during reading a record */ +} XLogReadRecordResult; + +/* + * internal state of XLogReadRecord + * + * XLogReadState runs a state machine while reading a record. Theses states + * are not seen outside the function. Each state may repeat several times + * exiting requesting caller for new data. See the comment of XLogReadRecrod + * for details. + */ +typedef enum XLogReadRecordState +{ + XLREAD_NEXT_RECORD, + XLREAD_TOT_LEN, + XLREAD_FIRST_FRAGMENT, + XLREAD_CONTINUATION +} XLogReadRecordState; + struct XLogReaderState { /* * Operational callbacks */ - XLogReaderRoutine routine; + WALSegmentCleanupCB cleanup_cb; /* ---------------------------------------- * Public parameters @@ -162,18 +137,14 @@ struct XLogReaderState */ uint64 system_identifier; - /* - * Opaque data for callbacks to use. Not used by XLogReader. - */ - void *private_data; - /* * Start and end point of last record read. EndRecPtr is also used as the * position to read next. Calling XLogBeginRead() sets EndRecPtr to the * starting position and ReadRecPtr to invalid. */ - XLogRecPtr ReadRecPtr; /* start of last record read */ + XLogRecPtr ReadRecPtr; /* start of last record read or being read */ XLogRecPtr EndRecPtr; /* end+1 of last record read */ + XLogRecPtr PrevRecPtr; /* start of previous record read */ /* * Set at the end of recovery: the start point of a partial record at the @@ -196,7 +167,9 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ char *readBuf; /* buffer to store data */ - bool page_verified; /* is the page on the buffer verified? */ + bool page_verified; /* is the page header on the buffer verified? */ + bool record_verified; /* is the current record header verified? */ + /* ---------------------------------------- * Decoded representation of current record @@ -237,8 +210,6 @@ struct XLogReaderState XLogRecPtr latestPagePtr; TimeLineID latestPageTLI; - /* beginning of the WAL record being read. */ - XLogRecPtr currRecPtr; /* timeline to read it from, 0 if a lookup is required */ TimeLineID currTLI; @@ -265,6 +236,15 @@ struct XLogReaderState char *readRecordBuf; uint32 readRecordBufSize; + /* + * XLogReadRecord() state + */ + XLogReadRecordState readRecordState; /* state machine state */ + int recordGotLen; /* amount of current record that has already + * been read */ + int recordRemainLen; /* length of current record that remains */ + XLogRecPtr recordContRecPtr; /* where the current record continues */ + /* Buffer to hold error message */ char *errormsg_buf; }; @@ -272,9 +252,7 @@ struct XLogReaderState /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, const char *waldir, - XLogReaderRoutine *routine, - void *private_data); -extern XLogReaderRoutine *LocalXLogReaderRoutine(void); + WALSegmentCleanupCB cleanup_cb); /* Free an XLogReader */ extern void XLogReaderFree(XLogReaderState *state); @@ -282,12 +260,19 @@ extern void XLogReaderFree(XLogReaderState *state); /* Position the XLogReader to given record */ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr); #ifdef FRONTEND -extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); +/* Function type definition for the read_page callback */ +typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader, + void *private); +extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, + XLogFindNextRecordCB read_page, void *private); #endif /* FRONTEND */ /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ -extern struct XLogRecord *XLogReadRecord(XLogReaderState *state, - char **errormsg); +extern XLogReadRecordResult XLogReadRecord(XLogReaderState *state, + XLogRecord **record, + char **errormsg); +/* Finalize function when the caller of XLogReadRecord failed */ +extern void XLogTerminateRead(XLogReaderState *state); /* Validate a page */ extern bool XLogReaderValidatePageHeader(XLogReaderState *state, @@ -307,6 +292,7 @@ typedef struct WALReadError } WALReadError; extern bool WALRead(XLogReaderState *state, + WALSegmentOpenCB segopenfn, WALSegmentCloseCB sgclosefn, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo); diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index 8669f7eeb3..e403d253d6 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -89,9 +89,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum, extern Relation CreateFakeRelcacheEntry(RelFileNode rnode); extern void FreeFakeRelcacheEntry(Relation fakerel); -extern bool read_local_xlog_page(XLogReaderState *state, - XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *cur_page); +extern bool read_local_xlog_page(XLogReaderState *state); extern void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p); diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index 614035e215..fecffdb3f6 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -390,7 +390,7 @@ * Enable debugging print statements for WAL-related operations; see * also the wal_debug GUC var. */ -/* #define WAL_DEBUG */ +#define WAL_DEBUG /* * Enable tracing of resource consumption during sort operations; diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index e0f513b773..3346188b48 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -29,6 +29,10 @@ typedef void (*LogicalOutputPluginWriterUpdateProgress) (struct LogicalDecodingC TransactionId xid ); +typedef struct LogicalDecodingContext LogicalDecodingContext; + +typedef bool (*LogicalDecodingXLogPageReadCB)(XLogReaderState *ctx); + typedef struct LogicalDecodingContext { /* memory context this is all allocated in */ @@ -39,6 +43,7 @@ typedef struct LogicalDecodingContext /* infrastructure pieces for decoding */ XLogReaderState *reader; + LogicalDecodingXLogPageReadCB page_read; struct ReorderBuffer *reorder; struct SnapBuild *snapshot_builder; @@ -115,14 +120,16 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin, List *output_plugin_options, bool need_full_snapshot, XLogRecPtr restart_lsn, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn, List *output_plugin_options, bool fast_forward, - XLogReaderRoutine *xl_routine, + LogicalDecodingXLogPageReadCB page_read, + WALSegmentCleanupCB cleanup_cb, LogicalOutputPluginWriterPrepareWrite prepare_write, LogicalOutputPluginWriterWrite do_write, LogicalOutputPluginWriterUpdateProgress update_progress); -- 2.27.0 From ee9199ce27cb11b5d0fc03f5dd4fdbee7a2a6d6d Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Thu, 30 Sep 2021 13:16:51 +0900 Subject: [PATCH v18 3/5] Remove globals readOff, readLen and readSegNo. The first two global variables are duplicated in XLogReaderState. Remove them, and also readSegNo, which should move into that struct too. Author: Kyotaro HORIGUCHI <horiguchi.kyotaro@lab.ntt.co.jp> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 316ba256f6..c4fe006776 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; /* * These variables are used similarly to the ones above, but for reading - * the XLOG. readOff is the offset of the page just read, readLen - * indicates how much of it has been read into readBuf, and readSource + * the XLOG. readOff is the offset of the page just read, readSource * indicates where we got the currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -911,10 +907,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static void XLogShutdownWalRcv(void); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); @@ -7894,7 +7892,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -12292,13 +12291,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -12311,7 +12311,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12319,10 +12319,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (ArchiveRecoveryRequested && IsUnderPostmaster) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12332,7 +12332,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12341,17 +12341,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->seg.ws_segno)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12379,40 +12376,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->seg.ws_segno, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->seg.ws_segno); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12447,23 +12440,23 @@ retry: * responsible for the validation. */ if (StandbyMode && - !XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + !XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { /* * Emit this error right now then retry this page immediately. Use * errmsg_internal() because the message was already translated. */ - if (xlogreader->errormsg_buf[0]) + if (state->errormsg_buf[0]) ereport(emode_for_corrupt_record(emode, EndRecPtr), - (errmsg_internal("%s", xlogreader->errormsg_buf))); + (errmsg_internal("%s", state->errormsg_buf))); /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12472,14 +12465,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12511,7 +12503,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; -- 2.27.0 From 8c6d4b5da61aeca9b12f8f8ea4b93ecc3f8aa769 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Thu, 30 Sep 2021 14:14:23 +0900 Subject: [PATCH v18 4/5] Make XLogFindNextRecord not use callback function The last function that uses page-read callback is XLogFindNextRecord. Lets make it free from call-back. This also simplifies the interface of WALDumpReadPage. --- src/backend/access/transam/xlogreader.c | 161 +++++++++++++----------- src/bin/pg_waldump/pg_waldump.c | 118 +++++++++-------- src/bin/pg_waldump/t/001_basic.pl | 35 +++++- src/include/access/xlogreader.h | 13 +- 4 files changed, 193 insertions(+), 134 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 4501a09245..e20c4dc4c7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -750,9 +750,9 @@ err: /* * Terminate read WAL. * - * When the caller failed to read the data requested from XLogReadRecord, it is - * supposed to call this function to set the correct reader state to reflect - * the failure. + * When the caller failed to read the data requested from XLogReadRecord or + * XLogFindNextRecord, it is supposed to call this function to set the correct + * reader state to reflect the failure. */ void XLogTerminateRead(XLogReaderState *state) @@ -1168,6 +1168,23 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * here. */ +XLogFindNextRecordState * +InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr) +{ + XLogFindNextRecordState *state = (XLogFindNextRecordState *) + palloc_extended(sizeof(XLogFindNextRecordState), + MCXT_ALLOC_NO_OOM | MCXT_ALLOC_ZERO); + if (!state) + return NULL; + + state->reader_state = reader_state; + state->targetRecPtr = start_ptr; + state->currRecPtr = start_ptr; + state->page_found = false; + + return state; +} + /* * Find the first record with an lsn >= RecPtr. * @@ -1179,123 +1196,115 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr, * This positions the reader, like XLogBeginRead(), so that the next call to * XLogReadRecord() will read the next valid record. */ -XLogRecPtr -XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, - XLogFindNextRecordCB read_page, void *private) +bool +XLogFindNextRecord(XLogFindNextRecordState *state) { - XLogRecPtr tmpRecPtr; - XLogRecPtr found = InvalidXLogRecPtr; XLogPageHeader header; XLogRecord *record; XLogReadRecordResult result; char *errormsg; - Assert(!XLogRecPtrIsInvalid(RecPtr)); + XLogRecPtr targetPagePtr; + int targetRecOff; + uint32 pageHeaderSize; /* * skip over potential continuation data, keeping in mind that it may span * multiple pages */ - tmpRecPtr = RecPtr; - while (true) +restart: + if (!state->page_found) { - XLogRecPtr targetPagePtr; - int targetRecOff; - uint32 pageHeaderSize; + Assert(!XLogRecPtrIsInvalid(state->currRecPtr)); /* * Compute targetRecOff. It should typically be equal or greater than * short page-header since a valid record can't start anywhere before * that, except when caller has explicitly specified the offset that * falls somewhere there or when we are skipping multi-page - * continuation record. It doesn't matter though because - * XLogNeedData() is prepared to handle that and will read at least - * short page-header worth of data + * continuation record. It doesn't matter though because XLogNeedData() + * is prepared to handle that and will read at least short page-header + * worth of data */ - targetRecOff = tmpRecPtr % XLOG_BLCKSZ; + targetRecOff = state->currRecPtr % XLOG_BLCKSZ; /* scroll back to page boundary */ - targetPagePtr = tmpRecPtr - targetRecOff; + targetPagePtr = state->currRecPtr - targetRecOff; - while (XLogNeedData(state, targetPagePtr, targetRecOff, - targetRecOff != 0)) - { - if (!read_page(state, private)) - break; - } + if (XLogNeedData(state->reader_state, targetPagePtr, targetRecOff, + targetRecOff != 0)) + return true; - if (!state->page_verified) + if (!state->reader_state->page_verified) goto err; - header = (XLogPageHeader) state->readBuf; + header = (XLogPageHeader) state->reader_state->readBuf; pageHeaderSize = XLogPageHeaderSize(header); /* we should have read the page header */ - Assert(state->readLen >= pageHeaderSize); + Assert(state->reader_state->readLen >= pageHeaderSize); - /* skip over potential continuation data */ if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) { - /* - * If the length of the remaining continuation data is more than - * what can fit in this page, the continuation record crosses over - * this page. Read the next page and try again. xlp_rem_len in the - * next page header will contain the remaining length of the - * continuation data - * - * Note that record headers are MAXALIGN'ed - */ - if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize)) - tmpRecPtr = targetPagePtr + XLOG_BLCKSZ; - else + if (MAXALIGN(header->xlp_rem_len) >= + (XLOG_BLCKSZ - pageHeaderSize)) { /* - * The previous continuation record ends in this page. Set - * tmpRecPtr to point to the first valid record + * If the length of the remaining continuation data is more + * than what can fit in this page, the continuation record + * crosses over this page. Read the next page and try + * again. xlp_rem_len in the next page header will contain the + * remaining length of the continuation data + * + * Note that record headers are MAXALIGN'ed */ - tmpRecPtr = targetPagePtr + pageHeaderSize - + MAXALIGN(header->xlp_rem_len); - break; + state->currRecPtr = targetPagePtr + XLOG_BLCKSZ; + goto restart; } + + /* + * The previous continuation record ends in this page. Set + * tmpRecPtr to point to the first valid record + */ + state->currRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(header->xlp_rem_len); } else - { - tmpRecPtr = targetPagePtr + pageHeaderSize; - break; - } + state->currRecPtr = targetPagePtr + pageHeaderSize; + + /* + * we know now that currRecPtr is an address pointing to a valid + * XLogRecord because either we're at the first record after the + * beginning of a page or we just jumped over the remaining data of + * a continuation. + */ + XLogBeginRead(state->reader_state, state->currRecPtr); + state->page_found = true; } - /* - * we know now that tmpRecPtr is an address pointing to a valid XLogRecord - * because either we're at the first record after the beginning of a page - * or we just jumped over the remaining data of a continuation. - */ - XLogBeginRead(state, tmpRecPtr); - while ((result = XLogReadRecord(state, &record, &errormsg)) != - XLREAD_FAIL) - { - if (result == XLREAD_NEED_DATA) - { - if (!read_page(state, private)) - goto err; - continue; - } + result = XLogReadRecord(state->reader_state, &record, &errormsg); - /* past the record we've found, break out */ - if (RecPtr <= state->ReadRecPtr) - { - /* Rewind the reader to the beginning of the last record. */ - found = state->ReadRecPtr; - XLogBeginRead(state, found); - return found; - } - } + if (result == XLREAD_FAIL) + goto err; + + if (result == XLREAD_NEED_DATA) + return true; + + /* past the record we've found, break out */ + if (state->reader_state->ReadRecPtr < state->targetRecPtr) + goto restart; + + /* Rewind the reader to the beginning of the last record. */ + state->currRecPtr = state->reader_state->ReadRecPtr; + XLogBeginRead(state->reader_state, state->currRecPtr); + return false; err: - XLogReaderInvalReadState(state); + XLogReaderInvalReadState(state->reader_state); - return InvalidXLogRecPtr; + state->currRecPtr = InvalidXLogRecPtr;; + return false; } #endif /* FRONTEND */ diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 80182621f8..d1d7427db0 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -29,14 +29,6 @@ static const char *progname; static int WalSegSz; -typedef struct XLogDumpPrivate -{ - TimeLineID timeline; - XLogRecPtr startptr; - XLogRecPtr endptr; - bool endptr_reached; -} XLogDumpPrivate; - typedef struct XLogDumpConfig { /* display options */ @@ -332,35 +324,40 @@ WALDumpCloseSegment(XLogReaderState *state) } /* - * pg_waldump's WAL page rader, also used as page_read callback for - * XLogFindNextRecord + * pg_waldump's WAL page rader + * + * timeline and startptr specifies the LSN, and reads up to endptr. */ static bool -WALDumpReadPage(XLogReaderState *state, void *priv) +WALDumpReadPage(XLogReaderState *state, TimeLineID timeline, + XLogRecPtr startptr, XLogRecPtr endptr) { XLogRecPtr targetPagePtr = state->readPagePtr; int reqLen = state->readLen; char *readBuff = state->readBuf; - XLogDumpPrivate *private = (XLogDumpPrivate *) priv; int count = XLOG_BLCKSZ; WALReadError errinfo; - if (private->endptr != InvalidXLogRecPtr) + /* determine the number of bytes to read on the page */ + if (endptr != InvalidXLogRecPtr) { - if (targetPagePtr + XLOG_BLCKSZ <= private->endptr) + if (targetPagePtr + XLOG_BLCKSZ <= endptr) count = XLOG_BLCKSZ; - else if (targetPagePtr + reqLen <= private->endptr) - count = private->endptr - targetPagePtr; + else if (targetPagePtr + reqLen <= endptr) + count = endptr - targetPagePtr; else { - private->endptr_reached = true; + /* Notify xlogreader that we didn't read at all */ state->readLen = -1; return false; } } + /* We should read more than requested by xlogreader */ + Assert(count >= state->readLen); + if (!WALRead(state, WALDumpOpenSegment, WALDumpCloseSegment, - readBuff, targetPagePtr, count, private->timeline, &errinfo)) + readBuff, targetPagePtr, count, timeline, &errinfo)) { WALOpenSegment *seg = &errinfo.wre_seg; char fname[MAXPGPATH]; @@ -380,7 +377,7 @@ WALDumpReadPage(XLogReaderState *state, void *priv) errinfo.wre_req); } - Assert(count >= state->readLen); + /* Notify xlogreader of how many bytes we have read */ state->readLen = count; return true; } @@ -774,7 +771,10 @@ main(int argc, char **argv) uint32 xlogid; uint32 xrecoff; XLogReaderState *xlogreader_state; - XLogDumpPrivate private; + XLogFindNextRecordState *findnext_state; + TimeLineID timeline; + XLogRecPtr startptr; + XLogRecPtr endptr; XLogDumpConfig config; XLogDumpStats stats; XLogRecord *record; @@ -820,14 +820,9 @@ main(int argc, char **argv) } } - memset(&private, 0, sizeof(XLogDumpPrivate)); - memset(&config, 0, sizeof(XLogDumpConfig)); - memset(&stats, 0, sizeof(XLogDumpStats)); - - private.timeline = 1; - private.startptr = InvalidXLogRecPtr; - private.endptr = InvalidXLogRecPtr; - private.endptr_reached = false; + timeline = 1; + startptr = InvalidXLogRecPtr; + endptr = InvalidXLogRecPtr; config.quiet = false; config.bkp_details = false; @@ -862,7 +857,7 @@ main(int argc, char **argv) optarg); goto bad_argument; } - private.endptr = (uint64) xlogid << 32 | xrecoff; + endptr = (uint64) xlogid << 32 | xrecoff; break; case 'f': config.follow = true; @@ -915,10 +910,10 @@ main(int argc, char **argv) goto bad_argument; } else - private.startptr = (uint64) xlogid << 32 | xrecoff; + startptr = (uint64) xlogid << 32 | xrecoff; break; case 't': - if (sscanf(optarg, "%d", &private.timeline) != 1) + if (sscanf(optarg, "%d", &timeline) != 1) { pg_log_error("could not parse timeline \"%s\"", optarg); goto bad_argument; @@ -995,21 +990,21 @@ main(int argc, char **argv) close(fd); /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &segno, WalSegSz); + XLogFromFileName(fname, &timeline, &segno, WalSegSz); - if (XLogRecPtrIsInvalid(private.startptr)) - XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, private.startptr); - else if (!XLByteInSeg(private.startptr, segno, WalSegSz)) + if (XLogRecPtrIsInvalid(startptr)) + XLogSegNoOffsetToRecPtr(segno, 0, WalSegSz, startptr); + else if (!XLByteInSeg(startptr, segno, WalSegSz)) { pg_log_error("start WAL location %X/%X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.startptr), + LSN_FORMAT_ARGS(startptr), fname); goto bad_argument; } /* no second file specified, set end position */ - if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(private.endptr)) - XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, private.endptr); + if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(endptr)) + XLogSegNoOffsetToRecPtr(segno + 1, 0, WalSegSz, endptr); /* parse ENDSEG if passed */ if (optind + 1 < argc) @@ -1025,26 +1020,26 @@ main(int argc, char **argv) close(fd); /* parse position from file */ - XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz); + XLogFromFileName(fname, &timeline, &endsegno, WalSegSz); if (endsegno < segno) fatal_error("ENDSEG %s is before STARTSEG %s", argv[optind + 1], argv[optind]); - if (XLogRecPtrIsInvalid(private.endptr)) + if (XLogRecPtrIsInvalid(endptr)) XLogSegNoOffsetToRecPtr(endsegno + 1, 0, WalSegSz, - private.endptr); + endptr); /* set segno to endsegno for check of --end */ segno = endsegno; } - if (!XLByteInSeg(private.endptr, segno, WalSegSz) && - private.endptr != (segno + 1) * WalSegSz) + if (!XLByteInSeg(endptr, segno, WalSegSz) && + endptr != (segno + 1) * WalSegSz) { pg_log_error("end WAL location %X/%X is not inside file \"%s\"", - LSN_FORMAT_ARGS(private.endptr), + LSN_FORMAT_ARGS(endptr), argv[argc - 1]); goto bad_argument; } @@ -1053,7 +1048,7 @@ main(int argc, char **argv) waldir = identify_target_directory(waldir, NULL); /* we don't know what to print */ - if (XLogRecPtrIsInvalid(private.startptr)) + if (XLogRecPtrIsInvalid(startptr)) { pg_log_error("no start WAL location given"); goto bad_argument; @@ -1068,27 +1063,40 @@ main(int argc, char **argv) if (!xlogreader_state) fatal_error("out of memory"); + findnext_state = + InitXLogFindNextRecord(xlogreader_state, startptr); + + if (!findnext_state) + fatal_error("out of memory"); + /* first find a valid recptr to start from */ - first_record = XLogFindNextRecord(xlogreader_state, private.startptr, - &WALDumpReadPage, (void*) &private); + while (XLogFindNextRecord(findnext_state)) + { + if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr)) + { + XLogTerminateRead(xlogreader_state); + break; + } + } + first_record = findnext_state->currRecPtr; if (first_record == InvalidXLogRecPtr) fatal_error("could not find a valid record after %X/%X", - LSN_FORMAT_ARGS(private.startptr)); + LSN_FORMAT_ARGS(startptr)); /* * Display a message that we're skipping data if `from` wasn't a pointer * to the start of a record and also wasn't a pointer to the beginning of * a segment (e.g. we were used in file mode). */ - if (first_record != private.startptr && - XLogSegmentOffset(private.startptr, WalSegSz) != 0) + if (first_record != startptr && + XLogSegmentOffset(startptr, WalSegSz) != 0) printf(ngettext("first record is after %X/%X, at %X/%X, skipping over %u byte\n", "first record is after %X/%X, at %X/%X, skipping over %u bytes\n", - (first_record - private.startptr)), - LSN_FORMAT_ARGS(private.startptr), + (first_record - startptr)), + LSN_FORMAT_ARGS(startptr), LSN_FORMAT_ARGS(first_record), - (uint32) (first_record - private.startptr)); + (uint32) (first_record - startptr)); for (;;) { @@ -1096,7 +1104,7 @@ main(int argc, char **argv) while (XLogReadRecord(xlogreader_state, &record, &errormsg) == XLREAD_NEED_DATA) { - if (!WALDumpReadPage(xlogreader_state, (void *) &private)) + if (!WALDumpReadPage(xlogreader_state, timeline, startptr, endptr)) { XLogTerminateRead(xlogreader_state); break; @@ -1105,7 +1113,7 @@ main(int argc, char **argv) if (!record) { - if (!config.follow || private.endptr_reached) + if (!config.follow) break; else { diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index fb2f807dc3..def8e03250 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -3,9 +3,42 @@ use strict; use warnings; +use PostgresNode; use TestLib; -use Test::More tests => 8; +use Test::More tests => 11; program_help_ok('pg_waldump'); program_version_ok('pg_waldump'); program_options_handling_ok('pg_waldump'); + +# Test: check if pg_waldump correctly skips over the contiulation +# pages while seeking for the first record. +my $node = PostgresNode->new('primary'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', 'wal_keep_size=1GB'); +$node->start; + +my $start_lsn = $node->safe_psql('postgres', "SELECT pg_current_wal_lsn()"); +my $start_file = $node->safe_psql('postgres', "SELECT pg_walfile_name(pg_current_wal_lsn())"); + +# insert a record spans over multiple pages +$node->safe_psql('postgres', + qq{SELECT pg_logical_emit_message(true, 'test 026', repeat('xyzxz', 123456))} +); + +# run pg_waldump from the second byte of a record +my ($file, $off) = split(/\//, $start_lsn); +my $target_lsn = sprintf("%X/%X", hex($file), hex($off) + 1); +my ($stdout, $stderr) = + run_command(["pg_waldump", '-s', $target_lsn, + $node->basedir . "/pgdata/pg_wal/" . $start_file]); + +ok ($stdout =~ + /first record is after ([0-9A-F\/]+), at ([0-9A-F\/]+), skipping over ([0-9]+) bytes/, + 'output contains required information'); +my $echoed_target = $1; +my $first_record = $2; +my $skipped_bytes = $3; + +ok ($echoed_target eq $target_lsn, 'target LSN is correct'); +ok ($skipped_bytes > 8192, 'skipped more than a page'); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index ff1aca719b..1a1e1939e0 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -56,6 +56,7 @@ typedef struct WALSegmentContext } WALSegmentContext; typedef struct XLogReaderState XLogReaderState; +typedef struct XLogFindNextRecordState XLogFindNextRecordState; /* Function type definition for the segment cleanup callback */ typedef void (*WALSegmentCleanupCB) (XLogReaderState *xlogreader); @@ -249,6 +250,14 @@ struct XLogReaderState char *errormsg_buf; }; +struct XLogFindNextRecordState +{ + XLogReaderState *reader_state; + XLogRecPtr targetRecPtr; + XLogRecPtr currRecPtr; + bool page_found; +}; + /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, const char *waldir, @@ -263,8 +272,8 @@ extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr); /* Function type definition for the read_page callback */ typedef bool (*XLogFindNextRecordCB) (XLogReaderState *xlogreader, void *private); -extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr, - XLogFindNextRecordCB read_page, void *private); +extern XLogFindNextRecordState *InitXLogFindNextRecord(XLogReaderState *reader_state, XLogRecPtr start_ptr); +extern bool XLogFindNextRecord(XLogFindNextRecordState *state); #endif /* FRONTEND */ /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ -- 2.27.0 From 44abe155352c549631ed71ceb4179572d7ded7b3 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi <horikyota.ntt@gmail.com> Date: Thu, 7 Oct 2021 09:56:45 +0900 Subject: [PATCH v18 5/5] Split readLen and reqLen of XLogReaderState. The variable is used as both out and in parameter of page-read functions. Separate the varialbe according to the roles. To avoid confusion between the two variables, provide a setter function for page-read functions to set readLen. --- src/backend/access/transam/xlog.c | 10 +++++----- src/backend/access/transam/xlogreader.c | 17 ++++++++--------- src/backend/access/transam/xlogutils.c | 6 +++--- src/backend/replication/walsender.c | 6 +++--- src/bin/pg_rewind/parsexlog.c | 12 +++++++----- src/bin/pg_waldump/pg_waldump.c | 6 +++--- src/include/access/xlogreader.h | 20 +++++++++++++++----- 7 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c4fe006776..4bbc4d3c6a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -12296,7 +12296,7 @@ XLogPageRead(XLogReaderState *state, { char *readBuf = state->readBuf; XLogRecPtr targetPagePtr = state->readPagePtr; - int reqLen = state->readLen; + int reqLen = state->reqLen; int readLen = 0; XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; @@ -12348,7 +12348,7 @@ retry: close(readFile); readFile = -1; readSource = XLOG_FROM_ANY; - state->readLen = -1; + XLogReaderNotifySize(state, -1); return false; } } @@ -12403,7 +12403,7 @@ retry: pgstat_report_wait_end(); Assert(targetSegNo == state->seg.ws_segno); - Assert(reqLen <= readLen); + Assert(readLen >= reqLen); state->seg.ws_tli = curFileTLI; @@ -12456,7 +12456,7 @@ retry: } Assert(state->readPagePtr == targetPagePtr); - state->readLen = readLen; + XLogReaderNotifySize(state, readLen); return true; next_record_is_invalid: @@ -12471,7 +12471,7 @@ next_record_is_invalid: if (StandbyMode) goto retry; - state->readLen = -1; + XLogReaderNotifySize(state, -1); return false; } diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index e20c4dc4c7..96fa1baf98 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -110,7 +110,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, WALOpenSegmentInit(&state->seg, &state->segcxt, wal_segment_size, waldir); - /* ReadRecPtr, EndRecPtr and readLen initialized to zeroes above */ + /* ReadRecPtr, EndRecPtr, reqLen and readLen initialized to zeroes above */ state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1, MCXT_ALLOC_NO_OOM); if (!state->errormsg_buf) @@ -264,12 +264,12 @@ XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr) * record being stored in *record. Otherwise *record is NULL. * * Returns XLREAD_NEED_DATA if more data is needed to finish reading the - * current record. In that case, state->readPagePtr and state->readLen inform + * current record. In that case, state->readPagePtr and state->reqLen inform * the desired position and minimum length of data needed. The caller shall * read in the requested data and set state->readBuf to point to a buffer * containing it. The caller must also set state->seg->ws_tli and * state->readLen to indicate the timeline that it was read from, and the - * length of data that is now available (which must be >= given readLen), + * length of data that is now available (which must be >= given reqLen), * respectively. * * If invalid data is encountered, returns XLREAD_FAIL with *record being set to @@ -662,7 +662,7 @@ restart: * XLogNeedData should have ensured that the whole page * header was read */ - Assert(state->readLen >= pageHeaderSize); + Assert(pageHeaderSize <= state->readLen); contdata = (char *) state->readBuf + pageHeaderSize; record_len = XLOG_BLCKSZ - pageHeaderSize; @@ -675,7 +675,7 @@ restart: * XLogNeedData should have ensured all needed data was * read */ - Assert(state->readLen >= request_len); + Assert(request_len <= state->readLen); memcpy(state->readRecordBuf + state->recordGotLen, (char *) contdata, record_len); @@ -728,7 +728,6 @@ restart: state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->segcxt.ws_segsize); } - Assert(!*record || state->readLen >= 0); if (DecodeXLogRecord(state, *record, errormsg)) return XLREAD_SUCCESS; @@ -824,7 +823,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, /* Request more data if we don't have the full header. */ if (state->readLen < pageHeaderSize) { - state->readLen = pageHeaderSize; + state->reqLen = pageHeaderSize; return true; } @@ -901,7 +900,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * will not come back here, but will request the actual target page. */ state->readPagePtr = pageptr - targetPageOff; - state->readLen = XLOG_BLCKSZ; + state->reqLen = XLOG_BLCKSZ; return true; } @@ -910,7 +909,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, * header so that we can validate it. */ state->readPagePtr = pageptr; - state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD); + state->reqLen = Max(reqLen + addLen, SizeOfXLogShortPHD); return true; } diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index fc09a72b8b..e69fd46099 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -848,7 +848,7 @@ bool read_local_xlog_page(XLogReaderState *state) { XLogRecPtr targetPagePtr = state->readPagePtr; - int reqLen = state->readLen; + int reqLen = state->reqLen; char *cur_page = state->readBuf; XLogRecPtr read_upto, loc; @@ -948,7 +948,7 @@ read_local_xlog_page(XLogReaderState *state) else if (targetPagePtr + reqLen > read_upto) { /* not enough data there */ - state->readLen = -1; + XLogReaderNotifySize(state, -1); return false; } else @@ -968,7 +968,7 @@ read_local_xlog_page(XLogReaderState *state) /* number of valid bytes in the buffer */ state->readPagePtr = targetPagePtr; - state->readLen = count; + XLogReaderNotifySize(state, count); return true; } diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 78d805ab80..81baeaded1 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -803,7 +803,7 @@ static bool logical_read_xlog_page(XLogReaderState *state) { XLogRecPtr targetPagePtr = state->readPagePtr; - int reqLen = state->readLen; + int reqLen = state->reqLen; char *cur_page = state->readBuf; XLogRecPtr flushptr; int count; @@ -822,7 +822,7 @@ logical_read_xlog_page(XLogReaderState *state) /* fail if not (implies we are going to shut down) */ if (flushptr < targetPagePtr + reqLen) { - state->readLen = -1; + XLogReaderNotifySize(state, -1); return false; } @@ -852,7 +852,7 @@ logical_read_xlog_page(XLogReaderState *state) XLByteToSeg(targetPagePtr, segno, state->segcxt.ws_segsize); CheckXLogRemoved(segno, state->seg.ws_tli); - state->readLen = count; + XLogReaderNotifySize(state, count); return true; } diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index da723e5340..119d9c5cd2 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -263,6 +263,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, XLogSegNo targetSegNo; int r; + Assert(xlogreader->reqLen <= XLOG_BLCKSZ); + XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz); XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd); targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz); @@ -313,7 +315,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, if (restoreCommand == NULL) { pg_log_error("could not open file \"%s\": %m", xlogfpath); - xlogreader->readLen = -1; + XLogReaderNotifySize(xlogreader, -1); return false; } @@ -328,7 +330,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, if (xlogreadfd < 0) { - xlogreader->readLen = -1; + XLogReaderNotifySize(xlogreader, -1); return false; } else @@ -346,7 +348,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0) { pg_log_error("could not seek in file \"%s\": %m", xlogfpath); - xlogreader->readLen = -1; + XLogReaderNotifySize(xlogreader, -1); return false; } @@ -360,14 +362,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, const char *datadir, pg_log_error("could not read file \"%s\": read %d of %zu", xlogfpath, r, (Size) XLOG_BLCKSZ); - xlogreader->readLen = -1; + XLogReaderNotifySize(xlogreader, -1); return false; } Assert(targetSegNo == xlogreadsegno); xlogreader->seg.ws_tli = targetHistory[*tliIndex].tli; - xlogreader->readLen = XLOG_BLCKSZ; + XLogReaderNotifySize(xlogreader, XLOG_BLCKSZ); return true; } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index d1d7427db0..7d782271fe 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -333,7 +333,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline, XLogRecPtr startptr, XLogRecPtr endptr) { XLogRecPtr targetPagePtr = state->readPagePtr; - int reqLen = state->readLen; + int reqLen = state->reqLen; char *readBuff = state->readBuf; int count = XLOG_BLCKSZ; WALReadError errinfo; @@ -348,7 +348,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline, else { /* Notify xlogreader that we didn't read at all */ - state->readLen = -1; + XLogReaderNotifySize(state, -1); return false; } } @@ -378,7 +378,7 @@ WALDumpReadPage(XLogReaderState *state, TimeLineID timeline, } /* Notify xlogreader of how many bytes we have read */ - state->readLen = count; + XLogReaderNotifySize(state, count); return true; } diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 1a1e1939e0..9efc04a4ad 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -159,18 +159,21 @@ struct XLogReaderState /* ---------------------------------------- * Communication with page reader - * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes. + * readBuf is XLOG_BLCKSZ bytes, valid up to at least reqLen bytes. * ---------------------------------------- */ /* variables to communicate with page reader */ XLogRecPtr readPagePtr; /* page pointer to read */ - int32 readLen; /* bytes requested to reader, or actual bytes - * read by reader, which must be larger than - * the request, or -1 on error */ + int32 reqLen; /* bytes requested to the caller */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ - bool record_verified; /* is the current record header verified? */ + bool record_verified;/* is the current record header verified? */ + /* variables to respond from the callers to xlogreader */ + int32 readLen; /* actual bytes read by reader, which must be + * larger than the request, or -1 on error. + * Use XLogReaderNotifyLength() to set a + * value. */ /* ---------------------------------------- * Decoded representation of current record @@ -258,6 +261,13 @@ struct XLogFindNextRecordState bool page_found; }; +/* setter functions of XLogReaderState used by other modules */ +static inline void +XLogReaderNotifySize(XLogReaderState *state, int32 len) +{ + state->readLen = len; +} + /* Get a new XLogReader */ extern XLogReaderState *XLogReaderAllocate(int wal_segment_size, const char *waldir, -- 2.27.0