Re: Upgrading rant.

On Fri, 03 Jan 2003 15:37:56 -0500, Tom Lane <tgl@sss.pgh.pa.us> wrote:
>The system tables are not the problem. [...]
>
>Changes in the on-disk representation of user tables would be harder to
>deal with, but they are also much rarer (AFAIR we've only done that
>twice: WAL required additions to page and tuple headers, and then there
>were Manfred's space-saving changes in 7.3).

So I'm the bad guy? ;-)  AFAICS handling the page and tuple format
changes doesn't need much more than what I have hacked together
yesterday afternoon:

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{Oid            t_oid;            /* OID of this tuple -- 4 bytes */CommandId    t_cmin;            /* insert CID stamp
--4 bytes each */CommandId    t_cmax;            /* delete CommandId stamp */TransactionId t_xmin;        /* insert XID
stamp-- 4 bytes each */TransactionId t_xmax;        /* delete XID stamp */ItemPointerData t_ctid;        /* current TID
ofthis or newer tuple */int16        t_natts;        /* number of attributes */uint16        t_infomask;        /*
variousinfos */uint8        t_hoff;            /* sizeof() tuple header *//* ^ - 31 bytes - ^ */bits8
t_bits[1];/*bit map of NULLs */
 
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{HeapTupleHeaderData newdata;Oid oid;HeapTupleHeader72 ht72;int len;
ht72 = (HeapTupleHeader72) ht;oid = ht72->t_oid;MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));
/* copy fixed fields */ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);newdata.t_natts =
ht72->t_natts;newdata.t_infomask= ht72->t_infomask;
 
HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);if (newdata.t_infomask & HEAP_XMAX_INVALID) {
HeapTupleHeaderSetCmin(&newdata,ht72->t_cmin);}/*if*/else {    HeapTupleHeaderSetXmax(&newdata,
ht72->t_xmax);}/*else*/
if (newdata.t_infomask & HEAP_MOVED) {    HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);}/*if*/else {
HeapTupleHeaderSetCmax(&newdata,ht72->t_cmax);}/*else*/
 
/* move new structure into original position */len = offsetof(HeapTupleHeaderData, t_bits);memcpy(ht, &newdata, len);
/* copy bitmap (if there is one) */if (ht->t_infomask & HEAP_HASNULL) {    int bitmaplen = BITMAPLEN(ht->t_natts);
intoff = offsetof(HeapTupleHeader72Data, t_bits);    char *p = (char *) ht;    int i;
 
    Assert(len < off);    for (i = 0; i < bitmaplen; ++i) {        p[len + i] = p[off + i];    }/*for*/
    len += bitmaplen;}/*if*/
/* pad rest with 0 */Assert(len < ht->t_hoff);memset((char *)ht + len, 0, ht->t_hoff - len);
/* change length, if requested */if (compact) {    if (oid != 0) {        len += sizeof(Oid);    }/*if*/
    ht->t_hoff = MAXALIGN(len);}/*if*/
/* copy oid (if there is one) */if (oid != 0) {    ht->t_infomask |= HEAP_HASOID;    HeapTupleHeaderSetOid(ht,
oid);}/*if*/
return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{PageHeader phdr = (PageHeader)page;int version = PageGetPageLayoutVersion(page);Size size = PageGetPageSize(page);int
maxoff= PageGetMaxOffsetNumber(page);int i;
 
if (version == PG_PAGE_LAYOUT_VERSION) {    /* already converted */    return;}/*if*/
Assert(version == 0);
for (i = 1; i <= maxoff; ++i) {    ItemId itid = PageGetItemId(page, i);    // ??? if (ItemIdIsUsed(itid)) ...
HeapTupleHeaderht = PageGetItem(page, itid);    Size oldsz = ht->t_hoff;    Size newsz;
 
    newsz = HeapTupleHeader_To73Format(ht, compact);    if (newsz < oldsz) {        int diff = oldsz - newsz;
ItemOffsetoff = ItemIdGetOffset(itid);        char *addr;        int lng;        int j;
 
        /* move tuple header to the right */        addr = (char *)ht;        memmove(addr + diff, addr, newsz);
itid->lp_off+= diff;        itid->lp_len -= diff;
 
        /*        ** Move all tuples that lie to the left of our tuple header.        ** (Shamelessly copied from
PageIndexTupleDelete()).       */        addr = (char *) page + phdr->pd_upper;        lng = (int) (off -
phdr->pd_upper);       if (lng > 0)            memmove(addr + diff, addr, lng);
 
        memset(addr, 0, diff);
        /*        ** Adjust upper free space boundary pointer,        ** lower is not affected.        */
phdr->pd_upper+= diff;
 
        /* Adjust linp entries. */        for (j = 1; j <= maxoff; ++j) {            ItemId ii = PageGetItemId(page,
j);           if (ii->lp_off < off)                ii->lp_off += diff;        }/*for*/    }/*if*/    else
Assert(newsz== oldsz);}/*for*/
 
PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{int version = PageGetPageLayoutVersion(page);Size size = PageGetPageSize(page);
if (version == PG_PAGE_LAYOUT_VERSION) {    /* already converted */    return;}/*if*/
Assert(version == 0);if (PageGetSpecialSize(page) == 0) {    /*    ** Heap page.    ** XXX Sure?    ** XXX Is there a
betterway to tell?    */    HeapPage_To73Format(page, true);}/*if*/else {    /*    ** Not a heap page: no format
change,just adjust version    */    PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);}/*else*/
 
}

This should handle all format changes I'm aware of:.  bitmap length.  overlaying fields.  optional oid.  page format
version
Am I missing something?

Above code is completely untested, I've not even run it through a
compiler; please consider it as a basis for discussion.  If there is
agreement, that we want 7.2 -> 7.3.x pg_upgrade, I'll put more work
into it.

What's missing is mainly a call to Page_To73Format() somewhere.  I can
think of.  an input-file-to-output-file-converter run by pg_upgrade instead of   copying/moving the files.  an
in-place-converterrun by pg_upgrade after copying/moving the   files.  converting each page during normal operation
immediatelyafter it   is fetched from disk.
 

#include <access/htup.h>

typedef struct HeapTupleHeader72Data
{Oid            t_oid;            /* OID of this tuple -- 4 bytes */CommandId    t_cmin;            /* insert CID stamp
--4 bytes each */CommandId    t_cmax;            /* delete CommandId stamp */TransactionId t_xmin;        /* insert XID
stamp-- 4 bytes each */TransactionId t_xmax;        /* delete XID stamp */ItemPointerData t_ctid;        /* current TID
ofthis or newer tuple */int16        t_natts;        /* number of attributes */uint16        t_infomask;        /*
variousinfos */uint8        t_hoff;            /* sizeof() tuple header *//* ^ - 31 bytes - ^ */bits8
t_bits[1];/*bit map of NULLs */
 
} HeapTupleHeader72Data;

typedef HeapTupleHeader72Data *HeapTupleHeader72;

/*
** Convert a pre-7.3 heap tuple header to 7.3 format.
**
** On entry ht points to a heap tuple header in 7.2 format,
** which will be converted to the new format in place.
** If compact is true, the size of the heap tuple header
** (t_hoff) is reduced, otherwise enough padding bytes are
** inserted to keep the old length.
**
** The return value is the new size.
*/
Size
HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact)
{HeapTupleHeaderData newdata;Oid oid;HeapTupleHeader72 ht72;int len;
ht72 = (HeapTupleHeader72) ht;oid = ht72->t_oid;MemSet(&newdata, 0, sizeof(HeapTupleHeaderData));
/* copy fixed fields */ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid);newdata.t_natts =
ht72->t_natts;newdata.t_infomask= ht72->t_infomask;
 
HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin);if (newdata.t_infomask & HEAP_XMAX_INVALID) {
HeapTupleHeaderSetCmin(&newdata,ht72->t_cmin);}/*if*/else {    HeapTupleHeaderSetXmax(&newdata,
ht72->t_xmax);}/*else*/
if (newdata.t_infomask & HEAP_MOVED) {    HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin);}/*if*/else {
HeapTupleHeaderSetCmax(&newdata,ht72->t_cmax);}/*else*/
 
/* move new structure into original position */len = offsetof(HeapTupleHeaderData, t_bits);memcpy(ht, &newdata, len);
/* copy bitmap (if there is one) */if (ht->t_infomask & HEAP_HASNULL) {    int bitmaplen = BITMAPLEN(ht->t_natts);
intoff = offsetof(HeapTupleHeader72Data, t_bits);    char *p = (char *) ht;    int i;
 
    Assert(len < off);    for (i = 0; i < bitmaplen; ++i) {        p[len + i] = p[off + i];    }/*for*/
    len += bitmaplen;}/*if*/
/* pad rest with 0 */Assert(len < ht->t_hoff);memset((char *)ht + len, 0, ht->t_hoff - len);
/* change length, if requested */if (compact) {    if (oid != 0) {        len += sizeof(Oid);    }/*if*/
    ht->t_hoff = MAXALIGN(len);}/*if*/
/* copy oid (if there is one) */if (oid != 0) {    ht->t_infomask |= HEAP_HASOID;    HeapTupleHeaderSetOid(ht,
oid);}/*if*/
return ht->t_hoff;
}

#include <storage/bufpage.h>
#include <access/htup.h>

/*
** Convert a pre 7.3 heap page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
HeapPage_To73Format(Page page, bool compact)
{PageHeader phdr = (PageHeader)page;int version = PageGetPageLayoutVersion(page);Size size = PageGetPageSize(page);int
maxoff= PageGetMaxOffsetNumber(page);int i;
 
if (version == PG_PAGE_LAYOUT_VERSION) {    /* already converted */    return;}/*if*/
Assert(version == 0);
for (i = 1; i <= maxoff; ++i) {    ItemId itid = PageGetItemId(page, i);    // ??? if (ItemIdIsUsed(itid)) ...
HeapTupleHeaderht = PageGetItem(page, itid);    Size oldsz = ht->t_hoff;    Size newsz;
 
    newsz = HeapTupleHeader_To73Format(ht, compact);    if (newsz < oldsz) {        int diff = oldsz - newsz;
ItemOffsetoff = ItemIdGetOffset(itid);        char *addr;        int lng;        int j;
 
        /* move tuple header to the right */        addr = (char *)ht;        memmove(addr + diff, addr, newsz);
itid->lp_off+= diff;        itid->lp_len -= diff;
 
        /*        ** Move all tuples that lie to the left of our tuple header.        ** (Shamelessly copied from
PageIndexTupleDelete()).       */        addr = (char *) page + phdr->pd_upper;        lng = (int) (off -
phdr->pd_upper);       if (lng > 0)            memmove(addr + diff, addr, lng);
 
        memset(addr, 0, diff);
        /*        ** Adjust upper free space boundary pointer,        ** lower is not affected.        */
phdr->pd_upper+= diff;
 
        /* Adjust linp entries. */        for (j = 1; j <= maxoff; ++j) {            ItemId ii = PageGetItemId(page,
j);           if (ii->lp_off < off)                ii->lp_off += diff;        }/*for*/    }/*if*/    else
Assert(newsz== oldsz);}/*for*/
 
PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);
}

/*
** Convert a pre 7.3 page to 7.3 format,
** or leave the page alone, if it is already in 7.3 format.
**
** The page is converted in place.
**
** We should have exclusive access to the page, either per
** LockBufferForCleanup() or because we a running in a standalone
** tool.
*/
void
Page_To73Format(Page page)
{int version = PageGetPageLayoutVersion(page);Size size = PageGetPageSize(page);
if (version == PG_PAGE_LAYOUT_VERSION) {    /* already converted */    return;}/*if*/
Assert(version == 0);if (PageGetSpecialSize(page) == 0) {    /*    ** Heap page.    ** XXX Sure?    ** XXX Is there a
betterway to tell?    */    HeapPage_To73Format(page, true);}/*if*/else {    /*    ** Not a heap page: no format
change,just adjust version    */    PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION);}/*else*/
 
}
ServusManfred