BUG #11280: Segmentation fault in dataPlaceToPageLeaf at gindatapage.c:645

The following bug has been logged on the website:

Bug reference:      11280
Logged by:          MÃ¥rten Svantesson
Email address:      marten.svantesson@ticket.se
PostgreSQL version: 9.4beta2
Operating system:   RHEL 6 kernel 2.6.32
Description:

As can be seen below one crash occurred during an insert to a table with a
gin index. (The table actually has 2 gin indexes.) The insert is done from
an import job where several inserts may be done in parallel. Also a similar
import is executed concurrently in another database of the cluster.

The crash hasn't happened to me with a clean index (reindex recently done).
So I can not give an exact step by step account of how to reproduce this.
The database was migrated from 9.3 with pg_update, but after that both
reindex and analyze full has been done.

As you can see auto explain is enabled, but I have experienced segmentation
faults in 9.4 without it enabled. This was before I had enabled core dumps
so if it was the exact same error I don't know. To get debug symbols I
compiled from the source tarball of 9.4beta2. Configure was run with

./configure --without-readline --enable-cassert --enable-debug CFLAGS="-ggdb
-fno-omit-frame-pointer"

The gin indexes on this table (and the whole database) are

CREATE INDEX offer_departurecode_o_badges_idx
  ON offer
  USING gin
  (departurecode_o, badges);

CREATE INDEX offer_badges_idx
  ON offer
  USING gin
  (badges);

where departurecode_o is defined as character varying(255) and badges as
bigint[]. The number of distinct values of departurecode_o is low: 26 in a
table with a bit over 2 million rows. The badges column is not present in
the insert.

# gdb /usr/pgsql-9.4/bin/postgres core.17901
<snip>
Core was generated by `postgres: test test_se xxx.xxx.xxx.xxx(41959) INSERT
'.
Program terminated with signal 11, Segmentation fault.
#0  0x0000000000473e04 in dataPlaceToPageLeaf (btree=0x7fff50edab40,
buf=66863, stack=0x2c520f0, insertdata=0x7fff50edab30,
prdata=0x7fff50edaa20, newlpage=0x7fff50edaa18, newrpage=0x7fff50edaa10) at
gindatapage.c:645
645                    segsize = SizeOfGinPostingList(lastleftinfo->seg);
Missing separate debuginfos, use: debuginfo-install
postgresql94-server-9.4beta2-1PGDG.rhel6.x86_64
(gdb) bt
#0  0x0000000000473e04 in dataPlaceToPageLeaf (btree=0x7fff50edab40,
buf=66863, stack=0x2c520f0, insertdata=0x7fff50edab30,
prdata=0x7fff50edaa20, newlpage=0x7fff50edaa18, newrpage=0x7fff50edaa10) at
gindatapage.c:645
#1  0x000000000047547f in dataPlaceToPage (btree=0x7fff50edab40, buf=66863,
stack=0x2c520f0, insertdata=0x7fff50edab30, updateblkno=4294967295,
prdata=0x7fff50edaa20, newlpage=0x7fff50edaa18, newrpage=0x7fff50edaa10) at
gindatapage.c:1180
#2  0x0000000000478478 in ginPlaceToPage (btree=0x7fff50edab40,
stack=0x2c520f0, insertdata=0x7fff50edab30, updateblkno=4294967295,
childbuf=0, buildStats=0x0) at ginbtree.c:354
#3  0x000000000047a70f in ginInsertValue (btree=0x7fff50edab40,
stack=0x2c520f0, insertdata=0x7fff50edab30, buildStats=0x0) at
ginbtree.c:743
#4  0x0000000000476fba in ginInsertItemPointers (index=0x7f0a3706fa10,
rootBlkno=351, items=0x7f0a342fe048, nitem=200383, buildStats=0x0) at
gindatapage.c:1882
#5  0x000000000046a27a in ginEntryInsert (ginstate=0x7fff50edaf90, attnum=2,
key=59, category=0 '\000', items=0x7f0a342fe048, nitem=200383,
buildStats=0x0) at gininsert.c:213
#6  0x00000000004859ec in ginInsertCleanup (ginstate=0x7fff50edaf90,
vac_delay=0 '\000', stats=0x0) at ginfast.c:836
#7  0x000000000048480e in ginHeapTupleFastInsert (ginstate=0x7fff50edaf90,
collector=0x7fff50edaf70) at ginfast.c:436
#8  0x000000000046ae44 in gininsert (fcinfo=0x7fff50edd5d0) at
gininsert.c:528
#9  0x00000000008e214e in FunctionCall6Coll (flinfo=0x2bdb748, collation=0,
arg1=139681849604624, arg2=140734551153248, arg3=140734551153216,
arg4=46313372, arg5=139681849559568, arg6=0) at fmgr.c:1437
#10 0x00000000004c36b6 in index_insert (indexRelation=0x7f0a3706fa10,
values=0x7fff50edda60, isnull=0x7fff50edda40 "", heap_t_ctid=0x2c2af9c,
heapRelation=0x7f0a37064a10, checkUnique=UNIQUE_CHECK_NO) at indexam.c:226
#11 0x000000000065082d in ExecInsertIndexTuples (slot=0x2bfdb80,
tupleid=0x2c2af9c, estate=0x2bfc238) at execUtils.c:1104
#12 0x0000000000664a56 in ExecInsert (slot=0x2bfdb80, planSlot=0x2bfdb80,
estate=0x2bfc238, canSetTag=1 '\001') at nodeModifyTable.c:274
#13 0x0000000000665c63 in ExecModifyTable (node=0x2bfc6f8) at
nodeModifyTable.c:1025
#14 0x000000000064240a in ExecProcNode (node=0x2bfc6f8) at
execProcnode.c:377
#15 0x0000000000640255 in ExecutePlan (estate=0x2bfc238,
planstate=0x2bfc6f8, operation=CMD_INSERT, sendTuples=1 '\001',
numberTuples=0, direction=ForwardScanDirection, dest=0x2b6ee78) at
execMain.c:1475
#16 0x000000000063e30f in standard_ExecutorRun (queryDesc=0x2c41160,
direction=ForwardScanDirection, count=0) at execMain.c:308
#17 0x00007f0ab6faa63b in ?? () from /usr/pgsql-9.4/lib/auto_explain.so
#18 0x000000000063e15e in ExecutorRun (queryDesc=0x2c41160,
direction=ForwardScanDirection, count=0) at execMain.c:254
#19 0x00000000007aae5b in ProcessQuery (plan=0x2c1a598, sourceText=0x2b6eb48
"insert into OFFER (availabilityCount, bookingLink, currency, description,
duration, arrivalCode_H, arrivalDate_H, departureCode_H, departureDate_H,
duration_H, flightNo_H, imported, numberOfAdults, nu"..., params=0x2c40cb8,
dest=0x2b6ee78, completionTag=0x7fff50ede070 "")
    at pquery.c:185
#20 0x00000000007ac77e in PortalRunMulti (portal=0x2be4038, isTopLevel=1
'\001', dest=0x2b6ee78, altdest=0xd5f0c0, completionTag=0x7fff50ede070 "")
at pquery.c:1279
#21 0x00000000007ac353 in FillPortalStore (portal=0x2be4038, isTopLevel=1
'\001') at pquery.c:1056
#22 0x00000000007abc96 in PortalRun (portal=0x2be4038,
count=9223372036854775807, isTopLevel=1 '\001', dest=0x2ba66e0,
altdest=0x2ba66e0, completionTag=0x7fff50ede280 "") at pquery.c:785
#23 0x00000000007a73df in exec_execute_message (portal_name=0x2ba62c8 "",
max_rows=9223372036854775807) at postgres.c:1928
#24 0x00000000007aa08c in PostgresMain (argc=1, argv=0x2b4ac28,
dbname=0x2b4a9f0 "charter_se", username=0x2b4a9d0 "reidar") at
postgres.c:4077
#25 0x000000000072e9a5 in BackendRun (port=0x2b68ea0) at postmaster.c:4117
#26 0x000000000072e0cf in BackendStartup (port=0x2b68ea0) at
postmaster.c:3791
#27 0x000000000072a8a9 in ServerLoop () at postmaster.c:1570
#28 0x0000000000729fd5 in PostmasterMain (argc=3, argv=0x2b49360) at
postmaster.c:1223
#29 0x000000000068ac67 in main (argc=3, argv=0x2b49360) at main.c:227

I have also gotten segmentation faults at the same code line in the auto
vacuum daemon. This can happen at the time of the same import giving the
segmentation fault above so it should involve the same indexes. The
segmentation fault in auto vacuum is actually much more frequent. It could
be related to that the badges column is bulk updated after the inserts. I
have seen a couple of different varieties of back traces:

(gdb) bt
#0  0x0000000000473e04 in dataPlaceToPageLeaf (btree=0x7fff5d4c71d0,
buf=81839, stack=0x167b6e0, insertdata=0x7fff5d4c71c0,
prdata=0x7fff5d4c70b0, newlpage=0x7fff5d4c70a8, newrpage=0x7fff5d4c70a0) at
gindatapage.c:645
#1  0x000000000047547f in dataPlaceToPage (btree=0x7fff5d4c71d0, buf=81839,
stack=0x167b6e0, insertdata=0x7fff5d4c71c0, updateblkno=4294967295,
prdata=0x7fff5d4c70b0, newlpage=0x7fff5d4c70a8, newrpage=0x7fff5d4c70a0) at
gindatapage.c:1180
#2  0x0000000000478478 in ginPlaceToPage (btree=0x7fff5d4c71d0,
stack=0x167b6e0, insertdata=0x7fff5d4c71c0, updateblkno=4294967295,
childbuf=0, buildStats=0x0) at ginbtree.c:354
#3  0x000000000047a70f in ginInsertValue (btree=0x7fff5d4c71d0,
stack=0x167b6e0, insertdata=0x7fff5d4c71c0, buildStats=0x0) at
ginbtree.c:743
#4  0x0000000000476fba in ginInsertItemPointers (index=0x7f82efa21358,
rootBlkno=55, items=0x17f46e8, nitem=84348, buildStats=0x0) at
gindatapage.c:1882
#5  0x000000000046a27a in ginEntryInsert (ginstate=0x7fff5d4c74c0, attnum=1,
key=59, category=0 '\000', items=0x17f46e8, nitem=84348, buildStats=0x0) at
gininsert.c:213
#6  0x00000000004859ec in ginInsertCleanup (ginstate=0x7fff5d4c74c0,
vac_delay=1 '\001', stats=0x0) at ginfast.c:836
#7  0x0000000000482503 in ginvacuumcleanup (fcinfo=0x7fff5d4c9ae0) at
ginvacuum.c:723
#8  0x00000000008e1cba in FunctionCall2Coll (flinfo=0x7fff5d4c9eb0,
collation=0, arg1=140734758690624, arg2=0) at fmgr.c:1324
#9  0x00000000004c4a74 in index_vacuum_cleanup (info=0x7fff5d4c9f40,
stats=0x0) at indexam.c:717
#10 0x00000000005be05b in do_analyze_rel (onerel=0x7f82efa08280,
vacstmt=0x7fff5d4ca350, acquirefunc=0x5bebf6 <acquire_sample_rows>,
relpages=440781, inh=0 '\000', elevel=13) at analyze.c:635
#11 0x00000000005bd2bc in analyze_rel (relid=31821, vacstmt=0x7fff5d4ca350,
bstrategy=0x163e070) at analyze.c:268
#12 0x0000000000634473 in vacuum (vacstmt=0x7fff5d4ca350, relid=31821,
do_toast=0 '\000', bstrategy=0x163e070, for_wraparound=0 '\000',
isTopLevel=1 '\001') at vacuum.c:254
#13 0x000000000071cd87 in autovacuum_do_vac_analyze (tab=0x163e188,
bstrategy=0x163e070) at autovacuum.c:2796
#14 0x000000000071c0bc in do_autovacuum () at autovacuum.c:2326
#15 0x000000000071af67 in AutoVacWorkerMain (argc=0, argv=0x0) at
autovacuum.c:1678
#16 0x000000000071ab61 in StartAutoVacWorker () at autovacuum.c:1463
#17 0x000000000072f393 in StartAutovacuumWorker () at postmaster.c:5188
#18 0x000000000072ec3f in sigusr1_handler (postgres_signal_arg=10) at
postmaster.c:4842
#19 <signal handler called>
#20 0x00000031742e15c3 in __select_nocancel () from /lib64/libc.so.6
#21 0x000000000072a778 in ServerLoop () at postmaster.c:1530
#22 0x0000000000729fd5 in PostmasterMain (argc=3, argv=0x1587360) at
postmaster.c:1223
#23 0x000000000068ac67 in main (argc=3, argv=0x1587360) at main.c:227

(gdb) bt
#0  0x0000000000473e04 in dataPlaceToPageLeaf (btree=0x7fff5d4c66f0,
buf=72419, stack=0x1620560, insertdata=0x7fff5d4c66e0,
prdata=0x7fff5d4c65d0, newlpage=0x7fff5d4c65c8, newrpage=0x7fff5d4c65c0) at
gindatapage.c:645
#1  0x000000000047547f in dataPlaceToPage (btree=0x7fff5d4c66f0, buf=72419,
stack=0x1620560, insertdata=0x7fff5d4c66e0, updateblkno=4294967295,
prdata=0x7fff5d4c65d0, newlpage=0x7fff5d4c65c8, newrpage=0x7fff5d4c65c0) at
gindatapage.c:1180
#2  0x0000000000478478 in ginPlaceToPage (btree=0x7fff5d4c66f0,
stack=0x1620560, insertdata=0x7fff5d4c66e0, updateblkno=4294967295,
childbuf=0, buildStats=0x0) at ginbtree.c:354
#3  0x000000000047a70f in ginInsertValue (btree=0x7fff5d4c66f0,
stack=0x1620560, insertdata=0x7fff5d4c66e0, buildStats=0x0) at
ginbtree.c:743
#4  0x0000000000476fba in ginInsertItemPointers (index=0x7f82efa0d108,
rootBlkno=55, items=0x7f82bf802048, nitem=141985, buildStats=0x0) at
gindatapage.c:1882
#5  0x000000000046a27a in ginEntryInsert (ginstate=0x7fff5d4c71f0, attnum=1,
key=59, category=0 '\000', items=0x7f82bf802048, nitem=141985,
buildStats=0x0) at gininsert.c:213
#6  0x00000000004859ec in ginInsertCleanup (ginstate=0x7fff5d4c71f0,
vac_delay=1 '\001', stats=0x15e2d90) at ginfast.c:836
#7  0x0000000000481e20 in ginbulkdelete (fcinfo=0x7fff5d4c9860) at
ginvacuum.c:608
#8  0x00000000008e1ee1 in FunctionCall4Coll (flinfo=0x7fff5d4c9c40,
collation=0, arg1=140734758690144, arg2=0, arg3=6524792, arg4=22948472) at
fmgr.c:1376
#9  0x00000000004c4936 in index_bulk_delete (info=0x7fff5d4c9d60, stats=0x0,
callback=0x638f78 <lazy_tid_reaped>, callback_state=0x15e2a78) at
indexam.c:690
#10 0x0000000000638619 in lazy_vacuum_index (indrel=0x7f82efa0d108,
stats=0x15e2bb0, vacrelstats=0x15e2a78) at vacuumlazy.c:1334
#11 0x0000000000637892 in lazy_scan_heap (onerel=0x7f82efa08280,
vacrelstats=0x15e2a78, Irel=0x15e2b10, nindexes=12, scan_all=0 '\000') at
vacuumlazy.c:1076
#12 0x0000000000636092 in lazy_vacuum_rel (onerel=0x7f82efa08280,
vacstmt=0x7fff5d4ca350, bstrategy=0x163fb90) at vacuumlazy.c:239
#13 0x0000000000635baf in vacuum_rel (relid=31821, vacstmt=0x7fff5d4ca350,
do_toast=0 '\000', for_wraparound=0 '\000') at vacuum.c:1285
#14 0x0000000000634429 in vacuum (vacstmt=0x7fff5d4ca350, relid=31821,
do_toast=0 '\000', bstrategy=0x163fb90, for_wraparound=0 '\000',
isTopLevel=1 '\001') at vacuum.c:237
#15 0x000000000071cd87 in autovacuum_do_vac_analyze (tab=0x163fca8,
bstrategy=0x163fb90) at autovacuum.c:2796
#16 0x000000000071c0bc in do_autovacuum () at autovacuum.c:2326
#17 0x000000000071af67 in AutoVacWorkerMain (argc=0, argv=0x0) at
autovacuum.c:1678
#18 0x000000000071ab61 in StartAutoVacWorker () at autovacuum.c:1463
#19 0x000000000072f393 in StartAutovacuumWorker () at postmaster.c:5188
#20 0x000000000072ec3f in sigusr1_handler (postgres_signal_arg=10) at
postmaster.c:4842
#21 <signal handler called>
#22 0x00000031742e15c3 in __select_nocancel () from /lib64/libc.so.6
#23 0x000000000072a778 in ServerLoop () at postmaster.c:1530
#24 0x0000000000729fd5 in PostmasterMain (argc=3, argv=0x1587360) at
postmaster.c:1223
#25 0x000000000068ac67 in main (argc=3, argv=0x1587360) at main.c:227