Re: gs_group_1 crashing on 13beta2/s390x

Re: To PostgreSQL Hackers
> On the Debian s390x buildd, the 13beta2 build is crashing:
> 
> 2020-07-15 01:19:59.149 UTC [859] LOG:  server process (PID 1415) was terminated by signal 11: Segmentation fault
> 2020-07-15 01:19:59.149 UTC [859] DETAIL:  Failed process was running: create table gs_group_1 as
>     select g100, g10, sum(g::numeric), count(*), max(g::text)
>     from gs_data_1 group by cube (g1000, g100,g10);

I wired gdb into the build process and got this backtrace:

2020-07-15 16:03:38.310 UTC [21073] LOG:  server process (PID 21575) was terminated by signal 11: Segmentation fault
2020-07-15 16:03:38.310 UTC [21073] DETAIL:  Failed process was running: create table gs_group_1 as
    select g100, g10, sum(g::numeric), count(*), max(g::text)
    from gs_data_1 group by cube (g1000, g100,g10);

******** build/src/bin/pg_upgrade/tmp_check/data.old/core ********
[New LWP 21575]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/s390x-linux-gnu/libthread_db.so.1".
Core was generated by `postgres: buildd regression [local] CREATE TABLE AS                           '.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  datumCopy (typByVal=false, typLen=-1, value=0) at ./build/../src/backend/utils/adt/datum.c:142
142            if (VARATT_IS_EXTERNAL_EXPANDED(vl))
#0  datumCopy (typByVal=false, typLen=-1, value=0) at ./build/../src/backend/utils/adt/datum.c:142
        vl = 0x0
        res = <optimized out>
        res = <optimized out>
        vl = <optimized out>
        eoh = <optimized out>
        resultsize = <optimized out>
        resultptr = <optimized out>
        realSize = <optimized out>
        resultptr = <optimized out>
        realSize = <optimized out>
        resultptr = <optimized out>
#1  datumCopy (value=0, typByVal=false, typLen=-1) at ./build/../src/backend/utils/adt/datum.c:131
        res = <optimized out>
        vl = <optimized out>
        eoh = <optimized out>
        resultsize = <optimized out>
        resultptr = <optimized out>
        realSize = <optimized out>
        resultptr = <optimized out>
#2  0x000002aa04423af8 in finalize_aggregate (aggstate=aggstate@entry=0x2aa05775920, peragg=peragg@entry=0x2aa056e02f0,
resultVal=0x2aa056e0208,resultIsNull=0x2aa056e022a, pergroupstate=<optimized out>, pergroupstate=<optimized out>) at
./build/../src/backend/executor/nodeAgg.c:1128
        fcinfodata = {fcinfo = {flinfo = 0x2aa056e0250, context = 0x2aa05775920, resultinfo = 0x0, fncollation = 0,
isnull= false, nargs = 1, args = 0x3fff6a7b578}, fcinfo_data = "\000\000\002\252\005n\002P\000\000\002\252\005wY ",
'\000'<repeats 13 times>,
"\247\000\001\000\000\002\252\005t\265\250\000\000\003\377\211\341\207F\000\000\003\377\000\000\002\001\000\000\000\000\000\000\003\376\000\000\000\000\000\000\017\370\000\000\000\000\000\000\001\377\000\000\000\000\000\000\000\260\000\000\000k\000\000\000k\000\000\000\000\000\000
\000\000\000\003\377\213\016J
\000\000\000p\000\000\000k\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\020",'\000' <repeats 11 times>,
"w\000\000\000|\000\000\000\000\000\000\000\002\000\000\002\252\006&9\250\000\000\002\252\005wZh\000\000\002\252\005wZH\000\000\003\377\213\n_\210\000\000\000\000\000\000\000\000\000\000"...}
        fcinfo = 0x3fff6a7b558
        anynull = <optimized out>
        oldContext = <optimized out>
        i = <optimized out>
        lc = <optimized out>
        pertrans = <error reading variable pertrans (value has been optimized out)>
#3  0x000002aa04423ff4 in finalize_aggregates (aggstate=aggstate@entry=0x2aa05775920,
peraggs=peraggs@entry=0x2aa056e0240,pergroup=0x2aa056c8ed8) at ./build/../src/backend/executor/nodeAgg.c:1345
 
        peragg = 0x2aa056e02f0
        transno = <optimized out>
        pergroupstate = 0x2aa056c8ef8
        econtext = <optimized out>
        aggvalues = 0x2aa056e01f8
        aggnulls = 0x2aa056e0228
        aggno = 2
        transno = <optimized out>
#4  0x000002aa04424f5c in agg_retrieve_direct (aggstate=0x2aa05775920) at
./build/../src/backend/executor/nodeAgg.c:2480
        econtext = 0x2aa05776080
        firstSlot = 0x2aa062639a8
        numGroupingSets = <optimized out>
        node = <optimized out>
        tmpcontext = 0x2aa05775d60
        peragg = 0x2aa056e0240
        outerslot = <optimized out>
        nextSetSize = <optimized out>
        pergroups = 0x2aa056c8ea8
        result = <optimized out>
        hasGroupingSets = <optimized out>
        currentSet = <optimized out>
        numReset = <optimized out>
        i = <optimized out>
        node = <optimized out>
        econtext = <optimized out>
        tmpcontext = <optimized out>
        peragg = <optimized out>
        pergroups = <optimized out>
        outerslot = <optimized out>
        firstSlot = <optimized out>
        result = <optimized out>
        hasGroupingSets = <optimized out>
        numGroupingSets = <optimized out>
        currentSet = <optimized out>
        nextSetSize = <optimized out>
        numReset = <optimized out>
        i = <optimized out>
#5  ExecAgg (pstate=0x2aa05775920) at ./build/../src/backend/executor/nodeAgg.c:2140
        node = 0x2aa05775920
        result = 0x0
#6  0x000002aa0441001a in ExecProcNode (node=0x2aa05775920) at ./build/../src/include/executor/executor.h:245
No locals.
#7  ExecutePlan (execute_once=<optimized out>, dest=0x2aa0565fa58, direction=<optimized out>, numberTuples=0,
sendTuples=<optimizedout>, operation=CMD_SELECT, use_parallel_mode=<optimized out>, planstate=0x2aa05775920,
estate=0x2aa057756f8)at ./build/../src/backend/executor/execMain.c:1646
 
        slot = <optimized out>
        current_tuple_count = 0
        slot = <optimized out>
        current_tuple_count = <optimized out>
#8  standard_ExecutorRun (queryDesc=0x2aa062df508, direction=<optimized out>, count=0, execute_once=<optimized out>) at
./build/../src/backend/executor/execMain.c:364
        estate = 0x2aa057756f8
        operation = CMD_SELECT
        dest = 0x2aa0565fa58
        sendTuples = <optimized out>
        oldcontext = 0x2aa0565f830
        __func__ = "standard_ExecutorRun"
#9  0x000002aa043933fa in ExecCreateTableAs (pstate=pstate@entry=0x2aa0565f948, stmt=stmt@entry=0x2aa055bea28,
params=params@entry=0x0,queryEnv=queryEnv@entry=0x0, qc=0x3fff6a7ca30) at
./build/../src/backend/commands/createas.c:354
        query = <optimized out>
        into = <optimized out>
        is_matview = <optimized out>
        dest = 0x2aa0565fa58
        save_userid = 0
        save_sec_context = 0
        save_nestlevel = 0
        address = {classId = 70885274, objectId = 1023, objectSubId = -156778696}
        rewritten = <optimized out>
        plan = 0x2aa062df3f8
        queryDesc = 0x2aa062df508
        __func__ = "ExecCreateTableAs"
#10 0x000002aa0459f378 in ProcessUtilitySlow (pstate=pstate@entry=0x2aa0565f948, pstmt=pstmt@entry=0x2aa055bead8,
queryString=queryString@entry=0x2aa055bcfb8"create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*),
max(g::text)\nfromgs_data_1 group by cube (g1000, g100,g10);", context=context@entry=PROCESS_UTILITY_TOPLEVEL,
params=params@entry=0x0,queryEnv=0x0, qc=0x3fff6a7ca30, dest=<error reading variable: value has been optimized out>) at
./build/../src/backend/tcop/utility.c:1600
        _save_exception_stack = 0x3fff6a7c820
        _save_context_stack = 0x0
        _local_sigjmp_buf = {{__jmpbuf = {{__gregs = {0, 2929258264904, 2929257605848, 0, 2929257605672, 2929257598904,
4396084256648,1, 18739560736704083, 18738779339296963}, __fpregs = {2929259797352, 2929259797352, 2929257598904,
2929258351376,0, 4397889735684, 4397889735216, 0}}}, __mask_was_saved = 0, __saved_mask = {__val = {4397889733480, 10,
2929257605400,0, 0, 2929257606120, 4096, 1, 0, 4396084256648, 2929242951088, 2929239505962, 4397889733120,
18736975094525723,2929257598904, 18446744069489843728}}}}
 
        _do_rethrow = false
        parsetree = 0x2aa055bea28
        isTopLevel = true
        isCompleteQuery = true
        needCleanup = false
        commandCollected = false
        address = {classId = 1023, objectId = 4138189632, objectSubId = 682}
        secondaryObject = {classId = 0, objectId = 0, objectSubId = 0}
        __func__ = "ProcessUtilitySlow"
#11 0x000002aa0459dd36 in standard_ProcessUtility (pstmt=0x2aa055bead8, queryString=0x2aa055bcfb8 "create table
gs_group_1as\nselect g100, g10, sum(g::numeric), count(*), max(g::text)\nfrom gs_data_1 group by cube (g1000,
g100,g10);",context=<optimized out>, params=0x0, queryEnv=<optimized out>, dest=0x2aa057d5b68, qc=0x3fff6a7ca30) at
./build/../src/backend/tcop/utility.c:1069
        parsetree = 0x2aa055bea28
        isTopLevel = <optimized out>
        isAtomicContext = <optimized out>
        pstate = 0x2aa0565f948
        readonly_flags = <optimized out>
        __func__ = "standard_ProcessUtility"
#12 0x000002aa0459e874 in ProcessUtility (pstmt=pstmt@entry=0x2aa055bead8, queryString=<optimized out>,
context=context@entry=PROCESS_UTILITY_TOPLEVEL,params=<optimized out>, queryEnv=queryEnv@entry=0x0, dest=0x2aa057d5b68,
qc=0x3fff6a7ca30)at ./build/../src/backend/tcop/utility.c:524
 
No locals.
#13 0x000002aa0459b210 in PortalRunUtility (portal=0x2aa05620008, pstmt=0x2aa055bead8, isTopLevel=<optimized out>,
setHoldSnapshot=<optimizedout>, dest=<optimized out>, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/pquery.c:1157
 
        utilityStmt = <optimized out>
        snapshot = 0x2aa05674c58
#14 0x000002aa0459bca0 in PortalRunMulti (portal=portal@entry=0x2aa05620008, isTopLevel=isTopLevel@entry=true,
setHoldSnapshot=setHoldSnapshot@entry=false,dest=<optimized out>, dest@entry=0x2aa057d5b68, altdest=<optimized out>,
altdest@entry=0x2aa057d5b68,qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/pquery.c:1303
 
        pstmt = <optimized out>
        stmtlist_item__state = {l = 0x2aa057d5b18, i = 0}
        active_snapshot_set = false
        stmtlist_item = 0x2aa057d5b30
#15 0x000002aa0459c9f4 in PortalRun (portal=portal@entry=0x2aa05620008, count=count@entry=9223372036854775807,
isTopLevel=isTopLevel@entry=true,run_once=run_once@entry=true, dest=dest@entry=0x2aa057d5b68, altdest=0x2aa057d5b68,
qc=0x3fff6a7ca30)at ./build/../src/backend/tcop/pquery.c:779
 
        _save_exception_stack = 0x3fff6a7ccc0
        _save_context_stack = 0x0
        _local_sigjmp_buf = {{__jmpbuf = {{__gregs = {2929258004488, 2, 2929258004488, 2929167695872, 2929257605768, 1,
4396084256648,4397889736544, 18739560736712267, 18738779339319315}, __fpregs = {89, 0, 2929257598904, 2929258351376,
4397889735216,4397889735684, 4397889735214, 0}}}, __mask_was_saved = 0, __saved_mask = {__val = {0, 8192, 2, 1, 1,
2929243650206,2929258004488, 2929257605768, 2, 4396084256648, 4397889736544, 2929237235946, 4397889734816,
4397889736544,2929239411110, 4397889734656}}}}
 
        _do_rethrow = <optimized out>
        result = <optimized out>
        nprocessed = <optimized out>
        saveTopTransactionResourceOwner = 0x2aa055e9bf0
        saveTopTransactionContext = 0x2aa05674b10
        saveActivePortal = 0x0
        saveResourceOwner = 0x2aa055e9bf0
        savePortalContext = 0x0
        saveMemoryContext = 0x2aa05674b10
        __func__ = "PortalRun"
#16 0x000002aa0459830a in exec_simple_query (query_string=<optimized out>) at
./build/../src/backend/tcop/postgres.c:1239
        snapshot_set = <optimized out>
        per_parsetree_context = <optimized out>
        plantree_list = <optimized out>
        parsetree = 0x2aa055bea58
        commandTag = <optimized out>
        qc = {commandTag = CMDTAG_UNKNOWN, nprocessed = 0}
        querytree_list = <optimized out>
        portal = 0x2aa05620008
        receiver = 0x2aa057d5b68
        format = 0
        parsetree_item__state = {l = 0x2aa055bea88, i = 0}
        dest = DestRemote
        oldcontext = <optimized out>
        parsetree_list = 0x2aa055bea88
        parsetree_item = 0x2aa055beaa0
        save_log_statement_stats = false
        was_logged = false
        use_implicit_block = false
        msec_str =
"\000\000\002\252\000\000\000Q\000\000\002\252\005[ϸ\000\000\002\252\000\000\000\000\000\000\003\377\213\n_\210"
        __func__ = "exec_simple_query"
#17 0x000002aa0459a05e in PostgresMain (argc=<optimized out>, argv=argv@entry=0x2aa055e8190, dbname=<optimized out>,
username=<optimizedout>) at ./build/../src/backend/tcop/postgres.c:4315
 
        query_string = 0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*),
max(g::text)\nfromgs_data_1 group by cube (g1000, g100,g10);"
 
        firstchar = 81
        input_message = {data = 0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*),
max(g::text)\nfromgs_data_1 group by cube (g1000, g100,g10);", len = 133, maxlen = 1024, cursor = 133}
 
        local_sigjmp_buf = {{__jmpbuf = {{__gregs = {8388608, 64, 2929244860962, 2929257583240, 2929244863032,
2929244863024,4396084256648, 4397889736544, 18739560736695925, 18738779339318659}, __fpregs = {4397889736544,
4397889736532,6, 4397419976768, 2929690573168, 2930182562592, 4397889736696, 0}}}, __mask_was_saved = 1, __saved_mask =
{__val= {0, 2929239505782, 2929257748144, 4397889736288, 4396084256648, 4397889736544, 2929242274994, 4397889735960,
1024,4396084256648, 4397889736544, 2929242135450, 4, 0, 4397889736320, 4397889736160}}}}
 
        send_ready_for_query = false
        disable_idle_in_transaction_timeout = false
        __func__ = "PostgresMain"
#18 0x000002aa04512066 in BackendRun (port=0x2aa055e16b0, port=0x2aa055e16b0) at
./build/../src/backend/postmaster/postmaster.c:4523
        av = 0x2aa055e8190
        maxac = <optimized out>
        ac = 1
        i = 1
        av = <optimized out>
        maxac = <optimized out>
        ac = <optimized out>
        i = <optimized out>
        __func__ = "BackendRun"
        __errno_location = <optimized out>
        __errno_location = <optimized out>
        __errno_location = <optimized out>
#19 BackendStartup (port=0x2aa055e16b0) at ./build/../src/backend/postmaster/postmaster.c:4215
        bn = <optimized out>
        pid = <optimized out>
        bn = <optimized out>
        pid = <optimized out>
        __func__ = "BackendStartup"
        __errno_location = <optimized out>
        __errno_location = <optimized out>
        save_errno = <optimized out>
        __errno_location = <optimized out>
        __errno_location = <optimized out>
#20 ServerLoop () at ./build/../src/backend/postmaster/postmaster.c:1727
        port = 0x2aa055e16b0
        i = <optimized out>
        rmask = {fds_bits = {32, 0 <repeats 15 times>}}
        selres = <optimized out>
        now = <optimized out>
        readmask = {fds_bits = {32, 0 <repeats 15 times>}}
        nSockets = 0
        last_lockfile_recheck_time = 1594829011
        last_touch_time = 1594829011
        __func__ = "ServerLoop"
#21 0x000002aa04513128 in PostmasterMain (argc=<optimized out>, argv=<optimized out>) at
./build/../src/backend/postmaster/postmaster.c:1400
        opt = <optimized out>
        status = <optimized out>
        userDoption = <optimized out>
        listen_addr_saved = false
        i = <optimized out>
        output_config_variable = <optimized out>
        __func__ = "PostmasterMain"
#22 0x000002aa04243fb4 in main (argc=<optimized out>, argv=0x2aa055b71b0) at ./build/../src/backend/main/main.c:210
        do_check_root = <optimized out>

Christoph