From e025bab12717f17bf551e5e9713a698e1b0f68d5 Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Thu, 4 Apr 2024 21:34:46 +0200 Subject: [PATCH v5 1/3] Add bitspace for more memory context types in MemoryChunk's hdrmask Assuming we don't want to use patterns from unused memory, common glibc malloc patterns, and wipe_mem-ed memory, we now have 18 new IDs available, from previously 0. In passing, simplify/clean up initialization of unused memory contexts in the mcxt_methods array, and update the naming scheme of unused and reserved MemoryContextMethodIDs so that we don't have to touch all lines to assign a new method ID, and that reserved IDs are clearer about why they're reserved. Inspiration: https://postgr.es/m/CAApHDvqGSpCU95TmM%3DBp%3D6xjL_nLys4zdZOpfNyWBk97Xrdj2w%40mail.gmail.com --- src/backend/utils/mmgr/mcxt.c | 63 ++++++++++++++++-------- src/include/utils/memutils_internal.h | 44 +++++++++++++++-- src/include/utils/memutils_memorychunk.h | 15 +++--- 3 files changed, 90 insertions(+), 32 deletions(-) diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 5d426795d9..de83a5b047 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -102,26 +102,49 @@ static const MemoryContextMethods mcxt_methods[] = { * seems sufficient to provide routines for the methods that might get * invoked from inspection of a chunk (see MCXT_METHOD calls below). */ - - [MCTX_UNUSED1_ID].free_p = BogusFree, - [MCTX_UNUSED1_ID].realloc = BogusRealloc, - [MCTX_UNUSED1_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED1_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED2_ID].free_p = BogusFree, - [MCTX_UNUSED2_ID].realloc = BogusRealloc, - [MCTX_UNUSED2_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED2_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED3_ID].free_p = BogusFree, - [MCTX_UNUSED3_ID].realloc = BogusRealloc, - [MCTX_UNUSED3_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED3_ID].get_chunk_space = BogusGetChunkSpace, - - [MCTX_UNUSED4_ID].free_p = BogusFree, - [MCTX_UNUSED4_ID].realloc = BogusRealloc, - [MCTX_UNUSED4_ID].get_chunk_context = BogusGetChunkContext, - [MCTX_UNUSED4_ID].get_chunk_space = BogusGetChunkSpace, +#define BOGUS_MCTX(id) \ + [id].free_p = BogusFree, \ + [id].realloc = BogusRealloc, \ + [id].get_chunk_context = BogusGetChunkContext, \ + [id].get_chunk_space = BogusGetChunkSpace + + BOGUS_MCTX(MCTX_7_UNUSED_ID), + BOGUS_MCTX(MCTX_8_UNUSED_ID), + BOGUS_MCTX(MCTX_11_UNUSED_ID), + BOGUS_MCTX(MCTX_12_UNUSED_ID), + BOGUS_MCTX(MCTX_13_UNUSED_ID), + BOGUS_MCTX(MCTX_14_UNUSED_ID), + BOGUS_MCTX(MCTX_15_UNUSED_ID), + BOGUS_MCTX(MCTX_16_UNUSED_ID), + BOGUS_MCTX(MCTX_19_UNUSED_ID), + BOGUS_MCTX(MCTX_20_UNUSED_ID), + BOGUS_MCTX(MCTX_21_UNUSED_ID), + BOGUS_MCTX(MCTX_22_UNUSED_ID), + BOGUS_MCTX(MCTX_23_UNUSED_ID), + BOGUS_MCTX(MCTX_24_UNUSED_ID), + BOGUS_MCTX(MCTX_27_UNUSED_ID), + BOGUS_MCTX(MCTX_28_UNUSED_ID), + BOGUS_MCTX(MCTX_29_UNUSED_ID), + BOGUS_MCTX(MCTX_30_UNUSED_ID), + /* + * Reserved IDs with bit patterns that we'd see if we were working on + * invalid memory (uninitialized or mem_wiped) + */ + BOGUS_MCTX(MCTX_RESERVED_UNUSEDMEM_ID), + BOGUS_MCTX(MCTX_RESERVED_WIPEMEM_ID), + /* + * Reserved IDs with bit patterns that we'd see if we were working with + * a glibc malloc chunk header, and not a memutils_memorychunk.h chunk. + */ + BOGUS_MCTX(MCTX_1_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_2_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_9_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_10_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_17_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_18_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_25_RESERVED_GLIBC_ID), + BOGUS_MCTX(MCTX_26_RESERVED_GLIBC_ID) +#undef BOGUS_MCTX }; /* diff --git a/src/include/utils/memutils_internal.h b/src/include/utils/memutils_internal.h index ad1048fd82..f6ec1962db 100644 --- a/src/include/utils/memutils_internal.h +++ b/src/include/utils/memutils_internal.h @@ -101,24 +101,58 @@ extern Size AlignedAllocGetChunkSpace(void *pointer); * malloc not palloc. (We can't tell that for most malloc implementations, * but it happens that glibc stores flag bits in the same place where we put * the MemoryContextMethodID, so the possible values are predictable for it.) + * + * MemoryContextMethodIDs that look like known likely bogus pointer patterns + * are reserved: + * - zero-ed and unused memory. + * - mem_wipe-d memory + * - malloc's 0b...001 and 0b...010 chunk patterns for normal and >128kB + * chunks, respectively. + * This leaves us with (2^n_bits) * (3/4) - 2 available IDs, or 22 options + * with the current allocation of 5 bits (see MEMORY_CONTEXT_METHODID_BITS). */ typedef enum MemoryContextMethodID { - MCTX_UNUSED1_ID, /* 000 occurs in never-used memory */ - MCTX_UNUSED2_ID, /* glibc malloc'd chunks usually match 001 */ - MCTX_UNUSED3_ID, /* glibc malloc'd chunks > 128kB match 010 */ + MCTX_RESERVED_UNUSEDMEM_ID, /* 0; occurs in never-used memory */ + MCTX_1_RESERVED_GLIBC_ID, /* 1=00001; glibc malloc'd chunks */ + MCTX_2_RESERVED_GLIBC_ID, /* 2=00010; glibc malloc'd chunks > 128kB */ MCTX_ASET_ID, MCTX_GENERATION_ID, MCTX_SLAB_ID, MCTX_ALIGNED_REDIRECT_ID, - MCTX_UNUSED4_ID, /* 111 occurs in wipe_mem'd memory */ + MCTX_7_UNUSED_ID, + MCTX_8_UNUSED_ID, + MCTX_9_RESERVED_GLIBC_ID, /* 9=01001; glibc malloc'd chunks */ + MCTX_10_RESERVED_GLIBC_ID, /* 10=01010; glibc malloc'd chunks > 128kB */ + MCTX_11_UNUSED_ID, + MCTX_12_UNUSED_ID, + MCTX_13_UNUSED_ID, + MCTX_14_UNUSED_ID, + MCTX_15_UNUSED_ID, + MCTX_16_UNUSED_ID, + MCTX_17_RESERVED_GLIBC_ID, /* 17=10001; glibc malloc'd chunk */ + MCTX_18_RESERVED_GLIBC_ID, /* 18=10010; glibc malloc'd chunk > 128kB */ + MCTX_19_UNUSED_ID, + MCTX_20_UNUSED_ID, + MCTX_21_UNUSED_ID, + MCTX_22_UNUSED_ID, + MCTX_23_UNUSED_ID, + MCTX_24_UNUSED_ID, + MCTX_25_RESERVED_GLIBC_ID, /* 25=11001; glibc malloc'd chunks */ + MCTX_26_RESERVED_GLIBC_ID, /* 26=11010; glibc malloc'd chunks > 128kB */ + MCTX_27_UNUSED_ID, + MCTX_28_UNUSED_ID, + MCTX_29_UNUSED_ID, + MCTX_30_UNUSED_ID, + MCTX_RESERVED_WIPEMEM_ID /* occurs in wipe_mem'd memory (0x7F) */ +#define MaxMemoryContextMethodID (MCTX_UNUSED28_ID) } MemoryContextMethodID; /* * The number of bits that 8-byte memory chunk headers can use to encode the * MemoryContextMethodID. */ -#define MEMORY_CONTEXT_METHODID_BITS 3 +#define MEMORY_CONTEXT_METHODID_BITS 5 #define MEMORY_CONTEXT_METHODID_MASK \ ((((uint64) 1) << MEMORY_CONTEXT_METHODID_BITS) - 1) diff --git a/src/include/utils/memutils_memorychunk.h b/src/include/utils/memutils_memorychunk.h index 38296abe1b..86ed28afab 100644 --- a/src/include/utils/memutils_memorychunk.h +++ b/src/include/utils/memutils_memorychunk.h @@ -25,14 +25,14 @@ * used to encode 4 separate pieces of information. Starting with the least * significant bits of 'hdrmask', the bit space is reserved as follows: * - * 1. 3-bits to indicate the MemoryContextMethodID as defined by + * 1. 5-bits to indicate the MemoryContextMethodID as defined by * MEMORY_CONTEXT_METHODID_MASK * 2. 1-bit to denote an "external" chunk (see below) * 3. 30-bits reserved for the MemoryContext to use for anything it * requires. Most MemoryContext likely want to store the size of the * chunk here. - * 4. 30-bits for the number of bytes that must be subtracted from the chunk - * to obtain the address of the block that the chunk is stored on. + * 4. 28-bits for the multiple of 4 bytes that must be subtracted from the + * chunk to obtain the address of the block that the chunk is stored on. * * In some cases, for example when memory allocations become large, it's * possible fields 3 and 4 above are not large enough to store the values @@ -91,7 +91,7 @@ * The maximum distance in bytes that a MemoryChunk can be offset from the * block that is storing the chunk. Must be 1 less than a power of 2. */ -#define MEMORYCHUNK_MAX_BLOCKOFFSET UINT64CONST(0x3FFFFFFF) +#define MEMORYCHUNK_MAX_BLOCKOFFSET UINT64CONST(0xFFFFFFF) /* define the least significant base-0 bit of each portion of the hdrmask */ #define MEMORYCHUNK_EXTERNAL_BASEBIT MEMORY_CONTEXT_METHODID_BITS @@ -135,7 +135,7 @@ typedef struct MemoryChunk * optimize out the & MEMORYCHUNK_MAX_BLOCKOFFSET. */ #define HdrMaskBlockOffset(hdrmask) \ - (((hdrmask) >> MEMORYCHUNK_BLOCKOFFSET_BASEBIT) & MEMORYCHUNK_MAX_BLOCKOFFSET) + ((((hdrmask) >> MEMORYCHUNK_BLOCKOFFSET_BASEBIT) & MEMORYCHUNK_MAX_BLOCKOFFSET) * 4) /* For external chunks only, check the magic number matches */ #define HdrMaskCheckMagic(hdrmask) \ @@ -157,11 +157,12 @@ MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block, Size blockoffset = (char *) chunk - (char *) block; Assert((char *) chunk >= (char *) block); - Assert(blockoffset <= MEMORYCHUNK_MAX_BLOCKOFFSET); + Assert((blockoffset % 4) == 0); + Assert((blockoffset / 4) <= MEMORYCHUNK_MAX_BLOCKOFFSET); Assert(value <= MEMORYCHUNK_MAX_VALUE); Assert((int) methodid <= MEMORY_CONTEXT_METHODID_MASK); - chunk->hdrmask = (((uint64) blockoffset) << MEMORYCHUNK_BLOCKOFFSET_BASEBIT) | + chunk->hdrmask = (((uint64) (blockoffset / 4)) << MEMORYCHUNK_BLOCKOFFSET_BASEBIT) | (((uint64) value) << MEMORYCHUNK_VALUE_BASEBIT) | methodid; } -- 2.40.1