I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.
This needs an additional option from autoconf, because pg_dump prints a
hex offset which needs to be either %Lx or %llx, but there is no hex
equivalent to INT64_FORMAT. To get this working I have arbitrarily
defined OFF_T_FORMATX as "%lld", but that needs to be changed before the
patch is incorporated.
Peter E suggested that there was no need for the macro definitions in
pg_backup.h but I wasn't clear how else to achieve what was needed.
--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18
Index: src/bin/pg_dump/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v
retrieving revision 1.36
diff -u -r1.36 Makefile
--- src/bin/pg_dump/Makefile 2002/07/27 20:10:05 1.36
+++ src/bin/pg_dump/Makefile 2002/08/15 10:31:53
@@ -18,6 +18,8 @@
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+CFLAGS += -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
all: submake-libpq submake-libpgport pg_dump pg_restore pg_dumpall
pg_dump: pg_dump.o common.o $(OBJS) $(libpq_builddir)/libpq.a
Index: src/bin/pg_dump/pg_backup.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup.h,v
retrieving revision 1.20
diff -u -r1.20 pg_backup.h
--- src/bin/pg_dump/pg_backup.h 2002/07/04 15:35:07 1.20
+++ src/bin/pg_dump/pg_backup.h 2002/08/15 10:31:53
@@ -27,6 +27,25 @@
#include "libpq-fe.h"
+/*
+ * Large file support
+ */
+
+#ifdef _LARGEFILE_SOURCE
+ #define FSEEK fseeko
+ #define FTELL ftello
+ #define OFF_T_FORMAT INT64_FORMAT
+ #define OFF_T_FORMATX "%llx" /* needs an autoconf setting */
+ typedef off_t OFF_T;
+#else
+ #define FSEEK fseek
+ #define FTELL ftell
+ #define OFF_T_FORMAT "%ld"
+ #define OFF_T_FORMATX "%lx"
+ typedef long int OFF_T;
+#endif
+
+
#define atooid(x) ((Oid) strtoul((x), NULL, 10))
#define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ? 1 : 0) )
#define oideq(x,y) ( (x) == (y) )
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.53
diff -u -r1.53 pg_backup_archiver.c
--- src/bin/pg_dump/pg_backup_archiver.c 2002/08/10 16:57:31 1.53
+++ src/bin/pg_dump/pg_backup_archiver.c 2002/08/15 10:31:55
@@ -1561,7 +1561,7 @@
}
/* If we can't seek, then mark the header as read */
- if (fseek(fh, 0, SEEK_SET) != 0)
+ if (FSEEK(fh, 0, SEEK_SET) != 0)
{
/*
* NOTE: Formats that use the looahead buffer can unset this in
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.19
diff -u -r1.19 pg_backup_custom.c
--- src/bin/pg_dump/pg_backup_custom.c 2002/05/29 01:38:56 1.19
+++ src/bin/pg_dump/pg_backup_custom.c 2002/08/15 10:31:56
@@ -81,13 +81,13 @@
char *zlibIn;
int inSize;
int hasSeek;
- int filePos;
+ OFF_T filePos;
int dataStart;
} lclContext;
typedef struct
{
- int dataPos;
+ OFF_T dataPos;
int dataLen;
} lclTocEntry;
@@ -99,7 +99,7 @@
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
-static int _getFilePos(ArchiveHandle *AH, lclContext *ctx);
+static OFF_T _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static char *modulename = gettext_noop("custom archiver");
@@ -188,7 +188,7 @@
if (!AH->FH)
die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
}
else
@@ -201,7 +201,7 @@
if (!AH->FH)
die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
ReadHead(AH);
ReadToc(AH);
@@ -285,7 +285,7 @@
{
lclTocEntry *ctx = (lclTocEntry *) te->formatData;
- ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);
+ ahprintf(AH, "-- Data Pos: " INT64_FORMAT " (Length %d)\n", ctx->dataPos, ctx->dataLen);
}
/*
@@ -486,7 +486,7 @@
/* Grab it */
- if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)
+ if (FSEEK(AH->FH, tctx->dataPos, SEEK_SET) != 0)
die_horribly(AH, modulename, "error during file seek: %s\n", strerror(errno));
_readBlockHeader(AH, &blkType, &id);
@@ -816,12 +816,12 @@
_CloseArchive(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
- int tpos;
+ OFF_T tpos;
if (AH->mode == archModeWrite)
{
WriteHead(AH);
- tpos = ftell(AH->FH);
+ tpos = FTELL(AH->FH);
WriteToc(AH);
ctx->dataStart = _getFilePos(AH, ctx);
WriteDataChunks(AH);
@@ -834,7 +834,7 @@
*/
if (ctx->hasSeek)
{
- fseek(AH->FH, tpos, SEEK_SET);
+ FSEEK(AH->FH, tpos, SEEK_SET);
WriteToc(AH);
}
}
@@ -853,14 +853,14 @@
/*
* Get the current position in the archive file.
*/
-static int
+static OFF_T
_getFilePos(ArchiveHandle *AH, lclContext *ctx)
{
- int pos;
+ OFF_T pos;
if (ctx->hasSeek)
{
- pos = ftell(AH->FH);
+ pos = FTELL(AH->FH);
if (pos != ctx->filePos)
{
write_msg(modulename, "WARNING: ftell mismatch with expected position -- ftell ignored\n");
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.16
diff -u -r1.16 pg_backup_files.c
--- src/bin/pg_dump/pg_backup_files.c 2002/05/29 01:38:56 1.16
+++ src/bin/pg_dump/pg_backup_files.c 2002/08/15 10:31:56
@@ -60,7 +60,7 @@
typedef struct
{
int hasSeek;
- int filePos;
+ OFF_T filePos;
FILE *blobToc;
} lclContext;
@@ -137,7 +137,7 @@
if (AH->FH == NULL)
die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
if (AH->compression < 0 || AH->compression > 9)
AH->compression = Z_DEFAULT_COMPRESSION;
@@ -155,7 +155,7 @@
if (AH->FH == NULL)
die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
ReadHead(AH);
ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.24
diff -u -r1.24 pg_backup_tar.c
--- src/bin/pg_dump/pg_backup_tar.c 2002/07/04 15:35:07 1.24
+++ src/bin/pg_dump/pg_backup_tar.c 2002/08/15 10:31:57
@@ -187,7 +187,7 @@
*/
/* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
- ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
if (AH->compression < 0 || AH->compression > 9)
AH->compression = Z_DEFAULT_COMPRESSION;
@@ -224,7 +224,7 @@
ctx->tarFHpos = 0;
- ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
/*
* Forcibly unmark the header as read since we use the lookahead
@@ -1011,9 +1011,9 @@
/*
* Find file len & go back to start.
*/
- fseek(tmp, 0, SEEK_END);
- th->fileLen = ftell(tmp);
- fseek(tmp, 0, SEEK_SET);
+ FSEEK(tmp, 0, SEEK_END);
+ th->fileLen = FTELL(tmp);
+ FSEEK(tmp, 0, SEEK_SET);
_tarWriteHeader(th);
@@ -1130,10 +1130,10 @@
while (!gotBlock)
{
#if 0
- if (ftell(ctx->tarFH) != ctx->tarFHpos)
+ if (FTELL(ctx->tarFH) != ctx->tarFHpos)
die_horribly(AH, modulename,
"mismatch in actual vs. predicted file position (%d vs. %d)\n",
- ftell(ctx->tarFH), ctx->tarFHpos);
+ FTELL(ctx->tarFH), ctx->tarFHpos);
#endif
/* Save the pos for reporting purposes */
@@ -1179,8 +1179,9 @@
if (chk != sum)
die_horribly(AH, modulename,
"corrupt tar header found in %s "
- "(expected %d (%o), computed %d (%o)) file position %ld (%lx)\n",
- &tag[0], sum, sum, chk, chk, ftell(ctx->tarFH), ftell(ctx->tarFH));
+ "(expected %d (%o), computed %d (%o)) file position "
+ OFF_T_FORMAT " (" OFF_T_FORMATX ")\n",
+ &tag[0], sum, sum, chk, chk, FTELL(ctx->tarFH), FTELL(ctx->tarFH));
th->targetFile = strdup(tag);
th->fileLen = len;