Re: Inefficient handling of LO-restore + Patch

Am Donnerstag, 11. April 2002 17:44 schrieb Tom Lane:
> "Mario Weilguni" <mario.weilguni@icomedias.com> writes:
> > And I did not find out how I can detect the large object
> > chunksize, either from getting it from the headers (include
> > "storage/large_object.h" did not work)
>

You did not answer if it's ok to post the patch, hope it's ok:
==================================
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c    Mon Feb 11
01:18:20 2002
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c    Thu Apr 11 10:41:09
2002
@@ -819,6 +819,9 @@        AH->createdBlobXref = 1;    }
+    /* Initialize the LO Buffer */
+    AH->lo_buf_used = 0;
+    /*     * Start long-running TXs if necessary     */
@@ -848,6 +851,19 @@voidEndRestoreBlob(ArchiveHandle *AH, Oid oid){
+        if(AH->lo_buf_used > 0) {
+      /* Write remaining bytes from the LO buffer */
+        int res;
+          res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf,
AH->lo_buf_used);
+
+      ahlog(AH, 5, "wrote remaining %d bytes of large object data (result =
%d)\n",
+            (int)AH->lo_buf_used, res);
+      if (res != AH->lo_buf_used)
+        die_horribly(AH, modulename, "could not write to large object (result: %d,
expected: %d)\n",
+             res, AH->lo_buf_used);
+          AH->lo_buf_used = 0;
+        }
+    lo_close(AH->connection, AH->loFd);    AH->writingBlob = 0;
@@ -1228,14 +1244,27 @@    if (AH->writingBlob)    {
-        res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
-        ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
-              (int) (size * nmemb), res);
-        if (res != size * nmemb)
+            if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
+          /* Split LO buffer */
+          int remaining = AH->lo_buf_size - AH->lo_buf_used;
+          int slack = nmemb * size - remaining;
+
+          memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
+          res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
+          ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+                      AH->lo_buf_size, res);
+          if (res != AH->lo_buf_size)            die_horribly(AH, modulename, "could not write to large object
(result:%d,  
expected: %d)\n",
-                         res, (int) (size * nmemb));
+                         res, AH->lo_buf_size);
+              memcpy(AH->lo_buf, ptr + remaining, slack);
+          AH->lo_buf_used = slack;
+           } else {
+             /* LO Buffer is still large enough, buffer it */
+         memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
+         AH->lo_buf_used += size * nmemb;
+           }
-        return res;
+           return size * nmemb;    }    else if (AH->gzOut)    {
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h
postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h    Mon Nov  5
18:46:30 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h    Thu Apr 11 10:41:14
2002
@@ -41,6 +41,7 @@#include <errno.h>#include "pqexpbuffer.h"
+#define LOBBUFSIZE 32768#ifdef HAVE_LIBZ#include <zlib.h>
@@ -240,6 +241,9 @@    RestoreOptions *ropt;        /* Used to check restore options in
*ahwrite etc */ 
+    void                    *lo_buf;
+    int                     lo_buf_used;
+    int                     lo_buf_size;} ArchiveHandle;typedef struct _tocEntry
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c    Wed Nov 28
00:48:12 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c    Thu Apr 11 10:42:45
2002
@@ -153,6 +153,12 @@    if (ctx->zp == NULL)        die_horribly(AH, modulename, "out of memory\n");
+    /* Initialize LO buffering */
+    AH->lo_buf_size = LOBBUFSIZE;
+    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+    if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+    /*     * zlibOutSize is the buffer size we tell zlib it can output to.  We     * actually allocate one extra byte
becausesome routines want to 
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c    Thu Oct 25
07:49:52 2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c    Thu Apr 11 10:43:01
2002
@@ -113,6 +113,12 @@    AH->formatData = (void *) ctx;    ctx->filePos = 0;
+    /* Initialize LO buffering */
+    AH->lo_buf_size = LOBBUFSIZE;
+    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+    if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+    /*     * Now open the TOC file     */
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c    Wed Jun 27 23:21:37
2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c    Thu Apr 11 10:44:53 2002
@@ -64,7 +64,6 @@     */    if (AH->mode == archModeRead)        die_horribly(AH, NULL, "this format cannot be
read\n");
-}/*
diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c
postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c
--- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c    Sun Oct 28 07:25:58
2001
+++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c    Thu Apr 11 10:44:08 2002
@@ -157,6 +157,12 @@    ctx = (lclContext *) malloc(sizeof(lclContext));    AH->formatData = (void *) ctx;
ctx->filePos= 0; 
+
+    /* Initialize LO buffering */
+    AH->lo_buf_size = LOBBUFSIZE;
+    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+    if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");    /*     * Now open the TOC file
============================

>
> ---------------------------(end of broadcast)---------------------------
> TIP 4: Don't 'kill -9' the postmaster