Обсуждение: Clog handling on exec'ed backend
Here is a another patch on the way to exec'ed Unix and Win32. It basically makes one of the Clog data structures into a shared memory area when using exec (EXEC_BACKEND is defined). It passes all the regression/initdb tests on Unix. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 359-1001 + If your life is a hard drive, | 13 Roberts Road + Christ can be your backup. | Newtown Square, Pennsylvania 19073 Index: src/backend/access/transam/clog.c =================================================================== RCS file: /cvsroot/pgsql-server/src/backend/access/transam/clog.c,v retrieving revision 1.14 diff -c -c -r1.14 clog.c *** src/backend/access/transam/clog.c 2 May 2003 21:59:31 -0000 1.14 --- src/backend/access/transam/clog.c 2 May 2003 22:09:26 -0000 *************** *** 157,163 **** * The value is automatically inherited by backends via fork, and * doesn't need to be in shared memory. */ ! static LWLockId ClogBufferLocks[NUM_CLOG_BUFFERS]; /* Per-buffer I/O locks */ /* * ClogDir is set during CLOGShmemInit and does not change thereafter. --- 157,163 ---- * The value is automatically inherited by backends via fork, and * doesn't need to be in shared memory. */ ! static LWLockId *ClogBufferLocks; /* Per-buffer I/O locks */ /* * ClogDir is set during CLOGShmemInit and does not change thereafter. *************** *** 271,311 **** /* * Initialization of shared memory for CLOG */ - int CLOGShmemSize(void) { ! return MAXALIGN(sizeof(ClogCtlData) + CLOG_BLCKSZ * NUM_CLOG_BUFFERS); } void CLOGShmemInit(void) { bool found; - char *bufptr; int slotno; /* this must agree with space requested by CLOGShmemSize() */ ! ClogCtl = (ClogCtlData *) ! ShmemInitStruct("CLOG Ctl", ! MAXALIGN(sizeof(ClogCtlData) + ! CLOG_BLCKSZ * NUM_CLOG_BUFFERS), ! &found); ! Assert(!found); ! memset(ClogCtl, 0, sizeof(ClogCtlData)); ! bufptr = ((char *) ClogCtl) + sizeof(ClogCtlData); ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) ! { ! ClogCtl->page_buffer[slotno] = bufptr; ! ClogCtl->page_status[slotno] = CLOG_PAGE_EMPTY; ! ClogBufferLocks[slotno] = LWLockAssign(); ! bufptr += CLOG_BLCKSZ; } ! /* ClogCtl->latest_page_number will be set later */ /* Init CLOG directory path */ snprintf(ClogDir, MAXPGPATH, "%s/pg_clog", DataDir); --- 271,337 ---- /* * Initialization of shared memory for CLOG */ int CLOGShmemSize(void) { ! return MAXALIGN(sizeof(ClogCtlData) + CLOG_BLCKSZ * NUM_CLOG_BUFFERS) ! #ifdef EXEC_BACKEND ! + MAXALIGN(NUM_CLOG_BUFFERS * sizeof(LWLockId)) ! #endif ! ; } + void CLOGShmemInit(void) { bool found; int slotno; + /* Handle ClogCtl */ + /* this must agree with space requested by CLOGShmemSize() */ ! ClogCtl = (ClogCtlData *) ShmemInitStruct("CLOG Ctl", ! MAXALIGN(sizeof(ClogCtlData) + ! CLOG_BLCKSZ * NUM_CLOG_BUFFERS), &found); ! ! if (!IsUnderPostmaster) ! /* Initialize ClogCtl shared memory area */ ! { ! char *bufptr; ! Assert(!found); ! memset(ClogCtl, 0, sizeof(ClogCtlData)); ! bufptr = (char *)ClogCtl + sizeof(ClogCtlData); ! ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) ! { ! ClogCtl->page_buffer[slotno] = bufptr; ! ClogCtl->page_status[slotno] = CLOG_PAGE_EMPTY; ! bufptr += CLOG_BLCKSZ; ! } ! ! /* ClogCtl->latest_page_number will be set later */ } + else + Assert(found); ! /* Handle ClogBufferLocks */ ! ! #ifdef EXEC_BACKEND ! ClogBufferLocks = (LWLockId *) ShmemInitStruct("CLOG Buffer Locks", ! NUM_CLOG_BUFFERS * sizeof(LWLockId), &found); ! Assert((!found && !IsUnderPostmaster) || (found && IsUnderPostmaster)); ! #else ! ClogBufferLocks = malloc(NUM_CLOG_BUFFERS * sizeof(LWLockId)); ! Assert(ClogBufferLocks); ! #endif ! ! if (!IsUnderPostmaster) ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) ! ClogBufferLocks[slotno] = LWLockAssign(); /* Init CLOG directory path */ snprintf(ClogDir, MAXPGPATH, "%s/pg_clog", DataDir); Index: src/backend/bootstrap/bootstrap.c =================================================================== RCS file: /cvsroot/pgsql-server/src/backend/bootstrap/bootstrap.c,v retrieving revision 1.150 diff -c -c -r1.150 bootstrap.c *** src/backend/bootstrap/bootstrap.c 2 May 2003 21:59:31 -0000 1.150 --- src/backend/bootstrap/bootstrap.c 2 May 2003 22:09:28 -0000 *************** *** 301,306 **** --- 301,311 ---- Assert(dbName); + if (IsUnderPostmaster && ExecBackend && MyProc /* ordinary backend */) + { + AttachSharedMemoryAndSemaphores(); + } + if (!IsUnderPostmaster) { if (!potential_DataDir) Index: src/backend/postmaster/postmaster.c =================================================================== RCS file: /cvsroot/pgsql-server/src/backend/postmaster/postmaster.c,v retrieving revision 1.319 diff -c -c -r1.319 postmaster.c *** src/backend/postmaster/postmaster.c 2 May 2003 22:02:47 -0000 1.319 --- src/backend/postmaster/postmaster.c 2 May 2003 22:09:30 -0000 *************** *** 172,177 **** --- 172,184 ---- static int ServerSock_UNIX = INVALID_SOCK; /* stream socket server */ #endif + /* Used to reduce macros tests */ + #ifdef EXEC_BACKEND + const bool ExecBackend = true; + #else + const bool ExecBackend = false; + #endif + /* * Set by the -o option */ *************** *** 1407,1413 **** elog(DEBUG1, "processCancelRequest: CheckPointPID in cancel request for process %d", backendPID); return; } ! /* See if we have a matching backend */ for (curr = DLGetHead(BackendList); curr; curr = DLGetSucc(curr)) --- 1414,1424 ---- elog(DEBUG1, "processCancelRequest: CheckPointPID in cancel request for process %d", backendPID); return; } ! else if (ExecBackend) ! { ! AttachSharedMemoryAndSemaphores(); ! } ! /* See if we have a matching backend */ for (curr = DLGetHead(BackendList); curr; curr = DLGetSucc(curr)) Index: src/backend/storage/ipc/ipci.c =================================================================== RCS file: /cvsroot/pgsql-server/src/backend/storage/ipc/ipci.c,v retrieving revision 1.51 diff -c -c -r1.51 ipci.c *** src/backend/storage/ipc/ipci.c 2 May 2003 21:59:31 -0000 1.51 --- src/backend/storage/ipc/ipci.c 2 May 2003 22:09:30 -0000 *************** *** 134,136 **** --- 134,149 ---- */ PMSignalInit(); } + + + /* + * AttachSharedMemoryAndSemaphores + * Attaches to the existing shared resources when exec()'d off + * by the postmaster. + */ + void + AttachSharedMemoryAndSemaphores(void) + { + CLOGShmemInit(); + } + Index: src/backend/tcop/postgres.c =================================================================== RCS file: /cvsroot/pgsql-server/src/backend/tcop/postgres.c,v retrieving revision 1.329 diff -c -c -r1.329 postgres.c *** src/backend/tcop/postgres.c 2 May 2003 21:59:31 -0000 1.329 --- src/backend/tcop/postgres.c 2 May 2003 22:09:32 -0000 *************** *** 1453,1458 **** --- 1453,1459 ---- break; } + /* * -d is not the same as setting * log_min_messages because it enables other *************** *** 1577,1582 **** --- 1578,1586 ---- * restart... */ } BaseInit(); + #ifdef EXECBACKEND + AttachSharedMemoryAndSemaphores(); + #endif } else { Index: src/include/miscadmin.h =================================================================== RCS file: /cvsroot/pgsql-server/src/include/miscadmin.h,v retrieving revision 1.120 diff -c -c -r1.120 miscadmin.h *** src/include/miscadmin.h 2 May 2003 21:59:31 -0000 1.120 --- src/include/miscadmin.h 2 May 2003 22:09:33 -0000 *************** *** 106,111 **** --- 106,112 ---- */ extern bool IsUnderPostmaster; extern bool ClientAuthInProgress; + extern const bool ExecBackend; extern int PostmasterMain(int argc, char *argv[]); extern void ClosePostmasterPorts(bool pgstat_too); Index: src/include/storage/ipc.h =================================================================== RCS file: /cvsroot/pgsql-server/src/include/storage/ipc.h,v retrieving revision 1.59 diff -c -c -r1.59 ipc.h *** src/include/storage/ipc.h 2 May 2003 21:59:31 -0000 1.59 --- src/include/storage/ipc.h 2 May 2003 22:09:33 -0000 *************** *** 32,36 **** --- 32,37 ---- extern void CreateSharedMemoryAndSemaphores(bool makePrivate, int maxBackends, int port); + extern void AttachSharedMemoryAndSemaphores(void); #endif /* IPC_H */
Bruce Momjian <pgman@candle.pha.pa.us> writes:
> It basically makes one of the Clog data structures into a shared memory
> area when using exec (EXEC_BACKEND is defined). It passes all the
> regression/initdb tests on Unix.
It would be cleaner to make the array of LWLockIds part of ClogCtlData,
if you're going to do it that way. I have doubts that pushing all
LWLockIds into shared memory is a feasible solution, however.
Out of curiosity: have you actually tested any of this Windows code
you're proposing to commit? A lot of it looks to me like it's not
gonna work --- stuff is getting done too soon or too late. For
example, I don't see how a backend is going to read GUC variables
when it doesn't yet know where the data directory is.
regards, tom lane
Tom Lane wrote: > Bruce Momjian <pgman@candle.pha.pa.us> writes: > > It basically makes one of the Clog data structures into a shared memory > > area when using exec (EXEC_BACKEND is defined). It passes all the > > regression/initdb tests on Unix. > > It would be cleaner to make the array of LWLockIds part of ClogCtlData, > if you're going to do it that way. I have doubts that pushing all > LWLockIds into shared memory is a feasible solution, however. OK, we will keep it as proposed so that shared memory issue only effects exec() setups. > Out of curiosity: have you actually tested any of this Windows code > you're proposing to commit? A lot of it looks to me like it's not > gonna work --- stuff is getting done too soon or too late. For > example, I don't see how a backend is going to read GUC variables > when it doesn't yet know where the data directory is. I have not tested it yet because it _all_ has to be done before anything is going to work. That's why I am trying exec() on Unix first as at least a minimal way to test things. I am going to have to move things around once I have the code working. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 359-1001 + If your life is a hard drive, | 13 Roberts Road + Christ can be your backup. | Newtown Square, Pennsylvania 19073
Committed --- I need to make more changes in this area in my next patch. --------------------------------------------------------------------------- Bruce Momjian wrote: > Here is a another patch on the way to exec'ed Unix and Win32. > > It basically makes one of the Clog data structures into a shared memory > area when using exec (EXEC_BACKEND is defined). It passes all the > regression/initdb tests on Unix. > > -- > Bruce Momjian | http://candle.pha.pa.us > pgman@candle.pha.pa.us | (610) 359-1001 > + If your life is a hard drive, | 13 Roberts Road > + Christ can be your backup. | Newtown Square, Pennsylvania 19073 > Index: src/backend/access/transam/clog.c > =================================================================== > RCS file: /cvsroot/pgsql-server/src/backend/access/transam/clog.c,v > retrieving revision 1.14 > diff -c -c -r1.14 clog.c > *** src/backend/access/transam/clog.c 2 May 2003 21:59:31 -0000 1.14 > --- src/backend/access/transam/clog.c 2 May 2003 22:09:26 -0000 > *************** > *** 157,163 **** > * The value is automatically inherited by backends via fork, and > * doesn't need to be in shared memory. > */ > ! static LWLockId ClogBufferLocks[NUM_CLOG_BUFFERS]; /* Per-buffer I/O locks */ > > /* > * ClogDir is set during CLOGShmemInit and does not change thereafter. > --- 157,163 ---- > * The value is automatically inherited by backends via fork, and > * doesn't need to be in shared memory. > */ > ! static LWLockId *ClogBufferLocks; /* Per-buffer I/O locks */ > > /* > * ClogDir is set during CLOGShmemInit and does not change thereafter. > *************** > *** 271,311 **** > /* > * Initialization of shared memory for CLOG > */ > - > int > CLOGShmemSize(void) > { > ! return MAXALIGN(sizeof(ClogCtlData) + CLOG_BLCKSZ * NUM_CLOG_BUFFERS); > } > > void > CLOGShmemInit(void) > { > bool found; > - char *bufptr; > int slotno; > > /* this must agree with space requested by CLOGShmemSize() */ > ! ClogCtl = (ClogCtlData *) > ! ShmemInitStruct("CLOG Ctl", > ! MAXALIGN(sizeof(ClogCtlData) + > ! CLOG_BLCKSZ * NUM_CLOG_BUFFERS), > ! &found); > ! Assert(!found); > > ! memset(ClogCtl, 0, sizeof(ClogCtlData)); > > ! bufptr = ((char *) ClogCtl) + sizeof(ClogCtlData); > > ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) > ! { > ! ClogCtl->page_buffer[slotno] = bufptr; > ! ClogCtl->page_status[slotno] = CLOG_PAGE_EMPTY; > ! ClogBufferLocks[slotno] = LWLockAssign(); > ! bufptr += CLOG_BLCKSZ; > } > > ! /* ClogCtl->latest_page_number will be set later */ > > /* Init CLOG directory path */ > snprintf(ClogDir, MAXPGPATH, "%s/pg_clog", DataDir); > --- 271,337 ---- > /* > * Initialization of shared memory for CLOG > */ > int > CLOGShmemSize(void) > { > ! return MAXALIGN(sizeof(ClogCtlData) + CLOG_BLCKSZ * NUM_CLOG_BUFFERS) > ! #ifdef EXEC_BACKEND > ! + MAXALIGN(NUM_CLOG_BUFFERS * sizeof(LWLockId)) > ! #endif > ! ; > } > > + > void > CLOGShmemInit(void) > { > bool found; > int slotno; > > + /* Handle ClogCtl */ > + > /* this must agree with space requested by CLOGShmemSize() */ > ! ClogCtl = (ClogCtlData *) ShmemInitStruct("CLOG Ctl", > ! MAXALIGN(sizeof(ClogCtlData) + > ! CLOG_BLCKSZ * NUM_CLOG_BUFFERS), &found); > ! > ! if (!IsUnderPostmaster) > ! /* Initialize ClogCtl shared memory area */ > ! { > ! char *bufptr; > > ! Assert(!found); > > ! memset(ClogCtl, 0, sizeof(ClogCtlData)); > > ! bufptr = (char *)ClogCtl + sizeof(ClogCtlData); > ! > ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) > ! { > ! ClogCtl->page_buffer[slotno] = bufptr; > ! ClogCtl->page_status[slotno] = CLOG_PAGE_EMPTY; > ! bufptr += CLOG_BLCKSZ; > ! } > ! > ! /* ClogCtl->latest_page_number will be set later */ > } > + else > + Assert(found); > > ! /* Handle ClogBufferLocks */ > ! > ! #ifdef EXEC_BACKEND > ! ClogBufferLocks = (LWLockId *) ShmemInitStruct("CLOG Buffer Locks", > ! NUM_CLOG_BUFFERS * sizeof(LWLockId), &found); > ! Assert((!found && !IsUnderPostmaster) || (found && IsUnderPostmaster)); > ! #else > ! ClogBufferLocks = malloc(NUM_CLOG_BUFFERS * sizeof(LWLockId)); > ! Assert(ClogBufferLocks); > ! #endif > ! > ! if (!IsUnderPostmaster) > ! for (slotno = 0; slotno < NUM_CLOG_BUFFERS; slotno++) > ! ClogBufferLocks[slotno] = LWLockAssign(); > > /* Init CLOG directory path */ > snprintf(ClogDir, MAXPGPATH, "%s/pg_clog", DataDir); > Index: src/backend/bootstrap/bootstrap.c > =================================================================== > RCS file: /cvsroot/pgsql-server/src/backend/bootstrap/bootstrap.c,v > retrieving revision 1.150 > diff -c -c -r1.150 bootstrap.c > *** src/backend/bootstrap/bootstrap.c 2 May 2003 21:59:31 -0000 1.150 > --- src/backend/bootstrap/bootstrap.c 2 May 2003 22:09:28 -0000 > *************** > *** 301,306 **** > --- 301,311 ---- > > Assert(dbName); > > + if (IsUnderPostmaster && ExecBackend && MyProc /* ordinary backend */) > + { > + AttachSharedMemoryAndSemaphores(); > + } > + > if (!IsUnderPostmaster) > { > if (!potential_DataDir) > Index: src/backend/postmaster/postmaster.c > =================================================================== > RCS file: /cvsroot/pgsql-server/src/backend/postmaster/postmaster.c,v > retrieving revision 1.319 > diff -c -c -r1.319 postmaster.c > *** src/backend/postmaster/postmaster.c 2 May 2003 22:02:47 -0000 1.319 > --- src/backend/postmaster/postmaster.c 2 May 2003 22:09:30 -0000 > *************** > *** 172,177 **** > --- 172,184 ---- > static int ServerSock_UNIX = INVALID_SOCK; /* stream socket server */ > #endif > > + /* Used to reduce macros tests */ > + #ifdef EXEC_BACKEND > + const bool ExecBackend = true; > + #else > + const bool ExecBackend = false; > + #endif > + > /* > * Set by the -o option > */ > *************** > *** 1407,1413 **** > elog(DEBUG1, "processCancelRequest: CheckPointPID in cancel request for process %d", backendPID); > return; > } > ! > /* See if we have a matching backend */ > > for (curr = DLGetHead(BackendList); curr; curr = DLGetSucc(curr)) > --- 1414,1424 ---- > elog(DEBUG1, "processCancelRequest: CheckPointPID in cancel request for process %d", backendPID); > return; > } > ! else if (ExecBackend) > ! { > ! AttachSharedMemoryAndSemaphores(); > ! } > ! > /* See if we have a matching backend */ > > for (curr = DLGetHead(BackendList); curr; curr = DLGetSucc(curr)) > Index: src/backend/storage/ipc/ipci.c > =================================================================== > RCS file: /cvsroot/pgsql-server/src/backend/storage/ipc/ipci.c,v > retrieving revision 1.51 > diff -c -c -r1.51 ipci.c > *** src/backend/storage/ipc/ipci.c 2 May 2003 21:59:31 -0000 1.51 > --- src/backend/storage/ipc/ipci.c 2 May 2003 22:09:30 -0000 > *************** > *** 134,136 **** > --- 134,149 ---- > */ > PMSignalInit(); > } > + > + > + /* > + * AttachSharedMemoryAndSemaphores > + * Attaches to the existing shared resources when exec()'d off > + * by the postmaster. > + */ > + void > + AttachSharedMemoryAndSemaphores(void) > + { > + CLOGShmemInit(); > + } > + > Index: src/backend/tcop/postgres.c > =================================================================== > RCS file: /cvsroot/pgsql-server/src/backend/tcop/postgres.c,v > retrieving revision 1.329 > diff -c -c -r1.329 postgres.c > *** src/backend/tcop/postgres.c 2 May 2003 21:59:31 -0000 1.329 > --- src/backend/tcop/postgres.c 2 May 2003 22:09:32 -0000 > *************** > *** 1453,1458 **** > --- 1453,1459 ---- > break; > } > > + > /* > * -d is not the same as setting > * log_min_messages because it enables other > *************** > *** 1577,1582 **** > --- 1578,1586 ---- > * restart... */ > } > BaseInit(); > + #ifdef EXECBACKEND > + AttachSharedMemoryAndSemaphores(); > + #endif > } > else > { > Index: src/include/miscadmin.h > =================================================================== > RCS file: /cvsroot/pgsql-server/src/include/miscadmin.h,v > retrieving revision 1.120 > diff -c -c -r1.120 miscadmin.h > *** src/include/miscadmin.h 2 May 2003 21:59:31 -0000 1.120 > --- src/include/miscadmin.h 2 May 2003 22:09:33 -0000 > *************** > *** 106,111 **** > --- 106,112 ---- > */ > extern bool IsUnderPostmaster; > extern bool ClientAuthInProgress; > + extern const bool ExecBackend; > > extern int PostmasterMain(int argc, char *argv[]); > extern void ClosePostmasterPorts(bool pgstat_too); > Index: src/include/storage/ipc.h > =================================================================== > RCS file: /cvsroot/pgsql-server/src/include/storage/ipc.h,v > retrieving revision 1.59 > diff -c -c -r1.59 ipc.h > *** src/include/storage/ipc.h 2 May 2003 21:59:31 -0000 1.59 > --- src/include/storage/ipc.h 2 May 2003 22:09:33 -0000 > *************** > *** 32,36 **** > --- 32,37 ---- > extern void CreateSharedMemoryAndSemaphores(bool makePrivate, > int maxBackends, > int port); > + extern void AttachSharedMemoryAndSemaphores(void); > > #endif /* IPC_H */ > > ---------------------------(end of broadcast)--------------------------- > TIP 5: Have you checked our extensive FAQ? > > http://www.postgresql.org/docs/faqs/FAQ.html -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 359-1001 + If your life is a hard drive, | 13 Roberts Road + Christ can be your backup. | Newtown Square, Pennsylvania 19073