Re: slowest tap tests - split or accelerate?

Andres Freund <andres@anarazel.de> writes:
> On 2022-01-19 11:54:01 -0500, Tom Lane wrote:
>> Me too ;-).  As I remarked earlier, I'd tried this once before and
>> gave up because it didn't seem to be winning much.  But that was
>> before we had so many initdb's triggered by TAP tests, I think.

> What approach did you use? Do you have a better idea than generating
> tmp_install/initdb_template?

No, it was largely the same as what you have here, I think.  I dug
up my WIP patch and attach it below, just in case there's any ideas
worth borrowing.

>> (Note that all four runs have the "fsync = on" removed from
>> 008_fsm_truncation.pl.)

> I assume you're planning on comitting that?

Yeah, will do that shortly.

            regards, tom lane

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index dc8a89af8e..a28a03ac72 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -332,6 +332,7 @@ ifeq ($(MAKELEVEL),0)
     rm -rf '$(abs_top_builddir)'/tmp_install
     $(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
     $(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install
>'$(abs_top_builddir)'/tmp_install/log/install.log2>&1 
+    '$(abs_top_builddir)/tmp_install$(bindir)/initdb' -D '$(abs_top_builddir)/tmp_install/proto_pgdata' --no-clean
--no-sync-A trust >'$(abs_top_builddir)'/tmp_install/log/initdb.log 2>&1 
 endif
     $(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra
DESTDIR='$(abs_top_builddir)'/tmp_installinstall >>'$(abs_top_builddir)'/tmp_install/log/install.log || exit; done) 
 endif
@@ -355,7 +356,7 @@ $(if $(filter $(PORTNAME),darwin),DYLD_LIBRARY_PATH,$(if $(filter $(PORTNAME),ai
 endef

 define with_temp_install
-PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" $(call
add_to_path,$(ld_library_path_var),$(abs_top_builddir)/tmp_install$(libdir))
+PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" $(call
add_to_path,$(ld_library_path_var),$(abs_top_builddir)/tmp_install$(libdir))
PG_PROTO_DATADIR='$(abs_top_builddir)/tmp_install/proto_pgdata'
 endef

 ifeq ($(enable_tap_tests),yes)
diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index f4fa600951..22c1a726c7 100644
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -404,8 +404,23 @@ sub init
     mkdir $self->backup_dir;
     mkdir $self->archive_dir;

-    TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
-        @{ $params{extra} });
+    # If we're using default initdb parameters, and the top-level "make check"
+    # created a prototype data directory for us, just copy that.
+    if (!defined($params{extra}) &&
+        defined($ENV{PG_PROTO_DATADIR}) &&
+        -d $ENV{PG_PROTO_DATADIR})
+    {
+        rmdir($pgdata);
+        RecursiveCopy::copypath($ENV{PG_PROTO_DATADIR}, $pgdata);
+        chmod(0700, $pgdata);
+    }
+    else
+    {
+        TestLib::system_or_bail('initdb', '-D', $pgdata,
+                                '--no-clean', '--no-sync', '-A', 'trust',
+                                @{ $params{extra} });
+    }
+
     TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata);

     open my $conf, '>>', "$pgdata/postgresql.conf";
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index abb742b1ed..04d7fb910b 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2214,6 +2214,8 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc

     if (temp_instance)
     {
+        char       *pg_proto_datadir;
+        struct stat ppst;
         FILE       *pg_conf;
         const char *env_wait;
         int            wait_seconds;
@@ -2243,20 +2245,43 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
         if (!directory_exists(buf))
             make_directory(buf);

-        /* initdb */
-        header(_("initializing database system"));
-        snprintf(buf, sizeof(buf),
-                 "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
-                 bindir ? bindir : "",
-                 bindir ? "/" : "",
-                 temp_instance,
-                 debug ? " --debug" : "",
-                 nolocale ? " --no-locale" : "",
-                 outputdir);
-        if (system(buf))
+        /* see if we should run initdb or just copy a prototype datadir */
+        pg_proto_datadir = getenv("PG_PROTO_DATADIR");
+        if (!nolocale &&
+            pg_proto_datadir &&
+            stat(pg_proto_datadir, &ppst) == 0 &&
+            S_ISDIR(ppst.st_mode))
         {
-            fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"),
progname,outputdir, buf); 
-            exit(2);
+            /* copy prototype */
+            header(_("copying prototype data directory"));
+            snprintf(buf, sizeof(buf),
+                     "cp -a \"%s\" \"%s/data\" > \"%s/log/initdb.log\" 2>&1",
+                     pg_proto_datadir,
+                     temp_instance,
+                     outputdir);
+            if (system(buf))
+            {
+                fprintf(stderr, _("\n%s: cp failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"),
progname,outputdir, buf); 
+                exit(2);
+            }
+        }
+        else
+        {
+            /* run initdb */
+            header(_("initializing database system"));
+            snprintf(buf, sizeof(buf),
+                     "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
+                     bindir ? bindir : "",
+                     bindir ? "/" : "",
+                     temp_instance,
+                     debug ? " --debug" : "",
+                     nolocale ? " --no-locale" : "",
+                     outputdir);
+            if (system(buf))
+            {
+                fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was:
%s\n"),progname, outputdir, buf); 
+                exit(2);
+            }
         }

         /*