Re: BUG #14835: Postgres crashed

Hello Nicola,

Weh have same the issues as you described on one of our databases .
It started with CentOS 6.8 and Postgres 9.5 - after upgrade to CentOS 6.9 ans Postgres 9.6 still same issues.

Did you find the cause or a fix for your crashes?

The database with issues is a poduction system. i've tried to trigger the chrash on a cloned system with no luck to reproduce.


Our Backtrace vom log:

2018-03-12 10:37:01.761 CET - OrderManagerServer 17.0.0.1 (PID: 15382)@10.40.1.162: LOG:  duration: 5207.109 ms  statement: WITH RECURSIVE sub2 AS ( WITH RECURSIVE sub AS ( SELECT id, order_entry_fk FROM order_entry WHERE (order_entry.ORDER_STATUS_FK <> 'CA' AND (order_entry.effective_when_begin < Cast( To_Timestamp('2018-03-12 23:59:59.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) AND order_entry.effective_when_end >= Cast( To_Timestamp('2018-03-12 00:00:00.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) )) UNION ALL SELECT parent.id, parent.order_entry_fk FROM order_entry AS parent JOIN sub AS child ON (parent.id = child.order_entry_fk) ) SELECT id, 1 AS level, CAST(id AS varchar) AS levelpath FROM order_entry WHERE order_entry_fk IS NULL AND id IN (SELECT id FROM sub WHERE sub.order_entry_fk IS NULL  )UNION ALL SELECT child.id, parent.level + 1 AS level, parent.levelpath || '.' || child.id AS levelpath FROM order_entry AS child JOIN sub2 AS parent ON  (child.order_entry_fk = parent.id) )SELECT (SELECT Count(*) FROM order_additional_info WHERE order_entry_fk=order_entry.id) AS order_additional_info_cnt, (SELECT Count(*) FROM orderer WHERE order_entry_fk=order_entry.id) AS orderer_cnt, LEVEL, order_entry.id, order_entry.external_id, order_entry.order_entry_fk , order_entry.order_status_fk , order_entry.service_location_fk, COALESCE(To_Char(order_entry.effective_when_begin, 'YYYYMMDDHH24MISS'), '00000000000000') AS effective_when_begin, COALESCE(To_Char(order_entry.effective_when_end, 'YYYYMMDDHH24MISS'), '00000000000000') AS effective_when_end, order_entry.modality_fk, order_entry.description , order_entry.code , order_entry.accession_number, order_entry.study_instance_uid , order_entry.series_instance_uid , order_entry.img_cnt, (SELECT domain FROM order_root WHERE order_entry_fk=order_entry.id) AS domain, (SELECT link_patient_visit_fk FROM order_root WHERE order_entry_fk=order_entry.id) AS link_patient_visit_fk, CASE WHEN (order_entry.ORDER_STATUS_FK <> 'CA' AND (order_entry.effective_when_begin < Cast( To_Timestamp('2018-03-12 23:59:59.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) AND order_entry.effective_when_end >= Cast( To_Timestamp('2018-03-12 00:00:00.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) )) THEN 1 ELSE 0 END AS exact_entry_match FROM sub2 JOIN order_entry ON order_entry.id = sub2.id ORDER BY levelpath
*** glibc detected *** postgres: synedra liaimdb 10.40.1.121(31908) SELECT: double free or corruption (!prev): 0x000000000429f510 ***
======= Backtrace: =========
/lib64/libc.so.6[0x30a7c75dee]
/lib64/libc.so.6[0x30a7c78c80]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecHashJoin+0x5a2)[0x5e1962]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0x208)[0x5ce388]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecScan+0x187)[0x5d56b7]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0x118)[0x5ce298]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecRecursiveUnion+0xad)[0x5ea09d]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0x188)[0x5ce308]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT[0x5ec3cc]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecScan+0x187)[0x5d56b7]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0xe8)[0x5ce268]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecHashJoin+0x2ad)[0x5e166d]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0x208)[0x5ce388]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecSort+0xf9)[0x5eba59]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(ExecProcNode+0x1e8)[0x5ce368]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(standard_ExecutorRun+0x18a)[0x5cbe7a]
/usr/pgsql-9.6/lib/pg_stat_statements.so(+0x44fb)[0x7f3006c294fb]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT[0x6e2fa7]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(PortalRun+0x188)[0x6e4178]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT[0x6e0ea1]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(PostgresMain+0xdcc)[0x6e256c]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(PostmasterMain+0x1875)[0x6823e5]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT(main+0x7a8)[0x609fe8]
/lib64/libc.so.6(__libc_start_main+0xfd)[0x30a7c1ed1d]
postgres: synedra liaimdb 10.40.1.121(31908) SELECT[0x46c039]
======= Memory map: ========
00400000-00a14000 r-xp 00000000 fd:00 684782                             /usr/pgsql-9.6/bin/postgres
00c14000-00c21000 rw-p 00614000 fd:00 684782                             /usr/pgsql-9.6/bin/postgres
00c21000-00c72000 rw-p 00000000 00:00 0
01f46000-01fab000 rw-p 00000000 00:00 0
01fab000-06b98000 rw-p 00000000 00:00 0
30a7800000-30a7820000 r-xp 00000000 fd:00 262157                         /lib64/ld-2.12.so (deleted)
30a7a1f000-30a7a21000 r--p 0001f000 fd:00 262157                         /lib64/ld-2.12.so (deleted)
30a7a21000-30a7a22000 rw-p 00021000 fd:00 262157                         /lib64/ld-2.12.so (deleted)
30a7a22000-30a7a23000 rw-p 00000000 00:00 0
30a7c00000-30a7d8a000 r-xp 00000000 fd:00 262181                         /lib64/libc-2.12.so (deleted)
30a7d8a000-30a7f8a000 ---p 0018a000 fd:00 262181                         /lib64/libc-2.12.so (deleted)
30a7f8a000-30a7f8e000 r--p 0018a000 fd:00 262181                         /lib64/libc-2.12.so (deleted)
30a7f8e000-30a7f90000 rw-p 0018e000 fd:00 262181                         /lib64/libc-2.12.so (deleted)
30a7f90000-30a7f94000 rw-p 00000000 00:00 0
30a8000000-30a8002000 r-xp 00000000 fd:00 262322                         /lib64/libdl-2.12.so (deleted)
30a8002000-30a8202000 ---p 00002000 fd:00 262322                         /lib64/libdl-2.12.so (deleted)
30a8202000-30a8203000 r--p 00002000 fd:00 262322                         /lib64/libdl-2.12.so (deleted)
30a8203000-30a8204000 rw-p 00003000 fd:00 262322                         /lib64/libdl-2.12.so (deleted)
30a8400000-30a8417000 r-xp 00000000 fd:00 262183                         /lib64/libpthread-2.12.so (deleted)
30a8417000-30a8617000 ---p 00017000 fd:00 262183                         /lib64/libpthread-2.12.so (deleted)
30a8617000-30a8618000 r--p 00017000 fd:00 262183                         /lib64/libpthread-2.12.so (deleted)
30a8618000-30a8619000 rw-p 00018000 fd:00 262183                         /lib64/libpthread-2.12.so (deleted)
30a8619000-30a861d000 rw-p 00000000 00:00 0
30a8800000-30a8807000 r-xp 00000000 fd:00 262287                         /lib64/librt-2.12.so (deleted)
30a8807000-30a8a06000 ---p 00007000 fd:00 262287                         /lib64/librt-2.12.so (deleted)
30a8a06000-30a8a07000 r--p 00006000 fd:00 262287                         /lib64/librt-2.12.so (deleted)
30a8a07000-30a8a08000 rw-p 00007000 fd:00 262287                         /lib64/librt-2.12.so (deleted)
30a8c00000-30a8c0c000 r-xp 00000000 fd:00 262360                         /lib64/libpam.so.0.82.2.#prelink#.BUCOSK (deleted)
30a8c0c000-30a8e0c000 ---p 0000c000 fd:00 262360                         /lib64/libpam.so.0.82.2.#prelink#.BUCOSK (deleted)
30a8e0c000-30a8e0d000 r--p 0000c000 fd:00 262360                         /lib64/libpam.so.0.82.2.#prelink#.BUCOSK (deleted)
30a8e0d000-30a8e0e000 rw-p 0000d000 fd:00 262360                         /lib64/libpam.so.0.82.2.#prelink#.BUCOSK (deleted)
30a9000000-30a9015000 r-xp 00000000 fd:00 262334                         /lib64/libz.so.1.2.3 (deleted)
30a9015000-30a9214000 ---p 00015000 fd:00 262334                         /lib64/libz.so.1.2.3 (deleted)
30a9214000-30a9215000 r--p 00014000 fd:00 262334                         /lib64/libz.so.1.2.3 (deleted)
30a9215000-30a9216000 rw-p 00015000 fd:00 262334                         /lib64/libz.so.1.2.3 (deleted)
30a9400000-30a941d000 r-xp 00000000 fd:00 262336                         /lib64/libselinux.so.1.#prelink#.wKSjfb (deleted)
30a941d000-30a961c000 ---p 0001d000 fd:00 262336                         /lib64/libselinux.so.1.#prelink#.wKSjfb (deleted)
30a961c000-30a961d000 r--p 0001c000 fd:00 262336                         /lib64/libselinux.so.1.#prelink#.wKSjfb (deleted)
30a961d000-30a961e000 rw-p 0001d000 fd:00 262336                         /lib64/libselinux.so.1.#prelink#.wKSjfb (deleted)
30a961e000-30a961f000 rw-p 00000000 00:00 0
30a9c00000-30a9c16000 r-xp 00000000 fd:00 262236                         /lib64/libresolv-2.12.so (deleted)
30a9c16000-30a9e16000 ---p 00016000 fd:00 262236                         /lib64/libresolv-2.12.so (deleted)
30a9e16000-30a9e17000 r--p 00016000 fd:00 262236                         /lib64/libresolv-2.12.so (deleted)
30a9e17000-30a9e18000 rw-p 00017000 fd:00 262236                         /lib64/libresolv-2.12.so (deleted)
30a9e18000-30a9e1a000 rw-p 00000000 00:00 0
30aa000000-30aa018000 r-xp 00000000 fd:00 262349                         /lib64/libaudit.so.1.0.0.#prelink#.JEMw5w (deleted)
30aa018000-30aa217000 ---p 00018000 fd:00 262349                         /lib64/libaudit.so.1.0.0.#prelink#.JEMw5w (deleted)
30aa217000-30aa219000 r--p 00017000 fd:00 262349                         /lib64/libaudit.so.1.0.0.#prelink#.JEMw5w (deleted)
30aa219000-30aa224000 rw-p 00019000 fd:00 262349                         /lib64/libaudit.so.1.0.0.#prelink#.JEMw5w (deleted)
30aa800000-30aa803000 r-xp 00000000 fd:00 263010                         /lib64/libplds4.so (deleted)
30aa803000-30aaa02000 ---p 00003000 fd:00 263010                         /lib64/libplds4.so (deleted)
0                         /lib64/libplds4.so (deleted)
30aaa03000-30aaa04000 rw-p 00003000 fd:00 263010                         /lib64/libplds4.so (deleted)
30aac00000-30aad3a000 r-xp 00000000 fd:00 679793                         /usr/lib64/libnss3.so (deleted)
30aad3a000-30aaf39000 ---p 0013a000 fd:00 679793                         /usr/lib64/libnss3.so (deleted)
30aaf39000-30aaf3f000 r--p 00139000 fd:00 679793                         /usr/lib64/libnss3.so (deleted)
30aaf3f000-30aaf41000 rw-p 0013f000 fd:00 679793                         /usr/lib64/libnss3.so (deleted)
30aaf41000-30aaf43000 rw-p 00000000 00:00 0
30ab000000-30ab004000 r-xp 00000000 fd:00 263009                         /lib64/libplc4.so (deleted)
30ab004000-30ab203000 ---p 00004000 fd:00 263009                         /lib64/libplc4.so (deleted)
30ab203000-30ab204000 r--p 00003000 fd:00 263009                         /lib64/libplc4.so (deleted)
30ab204000-30ab205000 rw-p 00004000 fd:00 263009                         /lib64/libplc4.so (deleted)
30ab400000-30ab439000 r-xp 00000000 fd:00 262256                         /lib64/libnspr4.so (deleted)
30ab439000-30ab639000 ---p 00039000 fd:00 262256                         /lib64/libnspr4.so (deleted)
30ab639000-30ab63a000 r--p 00039000 fd:00 262256                         /lib64/libnspr4.so (deleted)
30ab63a000-30ab63c000 rw-p 0003a000 fd:00 262256                         /lib64/libnspr4.so (deleted)
30ab63c000-30ab63e000 rw-p 00000000 00:00 0
30ab800000-30ab826000 r-xp 00000000 fd:00 659562                         /usr/lib64/libnssutil3.so (deleted)
30ab826000-30aba25000 ---p 00026000 fd:00 659562                         /usr/lib64/libnssutil3.so (deleted)
30aba25000-30aba2c000 r--p 00025000 fd:00 659562                         /usr/lib64/libnssutil3.so (deleted)
30aba2c000-30aba2d000 rw-p 0002c000 fd:00 659562                         /usr/lib64/libnssutil3.so (deleted)
30ac400000-30ac402000 r-xp 00000000 fd:00 262351                         /lib64/libfreebl3.so (deleted)
30ac402000-30ac601000 ---p 00002000 fd:00 262351                         /lib64/libfreebl3.so (deleted)
30ac601000-30ac602000 r--p 00001000 fd:00 262351                         /lib64/libfreebl3.so (deleted)
30ac602000-30ac603000 rw-p 00002000 fd:00 262351                         /lib64/libfreebl3.so (deleted)
30ac800000-30ac807000 r-xp 00000000 fd:00 262354                         /lib64/libcrypt-2.12.so (deleted)
30ac807000-30aca07000 ---p 00007000 fd:00 262354                         /lib64/libcrypt-2.12.so (deleted)
30aca07000-30aca08000 r--p 00007000 fd:00 262354                         /lib64/libcrypt-2.12.so (deleted)
30aca08000-30aca09000 rw-p 00008000 fd:00 262354                         /lib64/libcrypt-2.12.so (deleted)
30aca09000-30aca37000 rw-p 00000000 00:00 0
30ad000000-30ad028000 r-xp 00000000 fd:00 679812                         /usr/lib64/libsmime3.so (deleted)
30ad028000-30ad227000 ---p 00028000 fd:00 679812                         /usr/lib64/libsmime3.so (deleted)
30ad227000-30ad22b000 r--p 00027000 fd:00 679812                         /usr/lib64/libsmime3.so (deleted)
30ad22b000-30ad22c000 rw-p 0002b000 fd:00 679812                         /usr/lib64/libsmime3.so (deleted)
30ad400000-30ad447000 r-xp 00000000 fd:00 661533                         /usr/lib64/libssl3.so (deleted)
30ad447000-30ad647000 ---p 00047000 fd:00 661533                         /usr/lib64/libssl3.so (deleted)
30ad647000-30ad64b000 r--p 00047000 fd:00 661533                         /usr/lib64/libssl3.so (deleted)
30ad64b000-30ad64c000 rw-p 0004b000 fd:00 661533                         /usr/lib64/libssl3.so (deleted)
30ad64c000-30ad64d000 rw-p 00000000 00:00 0
30adc00000-30add49000 r-xp 00000000 fd:00 659737                         /usr/lib64/libxml2.so.2.7.6.#prelink#.NfqqBy (deleted)
30add49000-30adf48000 ---p 00149000 fd:00 659737                         /usr/lib64/libxml2.so.2.7.6.#prelink#.NfqqBy (deleted)
30adf48000-30adf52000 rw-p 00148000 fd:00 659737                         /usr/lib64/libxml2.so.2.7.6.#prelink#.NfqqBy (deleted)
30adf52000-30adf53000 rw-p 00000000 00:00 0
30ae800000-30ae803000 r-xp 00000000 fd:00 263020                         /lib64/libcom_err.so.2.1.#prelink#.coCiX7 (deleted)
30ae803000-30aea02000 ---p 00003000 fd:00 263020                         /lib64/libcom_err.so.2.1.#prelink#.coCiX7 (deleted)
30aea02000-30aea03000 r--p 00002000 fd:00 263020                         /lib64/libcom_err.so.2.1.#prelink#.coCiX7 (deleted)
30aea03000-30aea04000 rw-p 00003000 fd:00 263020                         /lib64/libcom_err.so.2.1.#prelink#.coCiX7 (deleted)
30af400000-30af419000 r-xp 00000000 fd:00 679813                         /usr/lib64/libsasl2.so.2.0.23.#prelink#.iqyza0 (deleted)
30af419000-30af618000 ---p 00019000 fd:00 679813                         /usr/lib64/libsasl2.so.2.0.23.#prelink#.iqyza0 (deleted)
30af618000-30af619000 r--p 00018000 fd:00 679813                         /usr/lib64/libsasl2.so.2.0.23.#prelink#.iqyza0 (deleted)
30af619000-30af61a000 rw-p 00019000 fd:00 679813                         /usr/lib64/libsasl2.so.2.0.23.#prelink#.iqyza0 (deleted)
30b0000000-30b000e000 r-xp 00000000 fd:00 262275                         /lib64/liblber-2.4.so.2.10.3.#prelink#.4gLtMe (deleted)
30b000e000-30b020d000 ---p 0000e000 fd:00 262275                         /lib64/liblber-2.4.so.2.10.3.#prelink#.4gLtMe (deleted)
30b020d000-30b020e000 r--p 0000d000 fd:00 262275                         /lib64/liblber-2.4.so.2.10.3.#prelink#.4gLtMe (deleted)
30b020e000-30b020f000 rw-p 0000e000 fd:00 262275                         /lib64/liblber-2.4.so.2.10.3.#prelink#.4gLtMe (deleted)
30b0400000-30b04dc000 r-xp 00000000 fd:00 263021                         /lib64/libkrb5.so.3.3 (deleted)
30b04dc000-30b06db000 ---p 000dc000 fd:00 263021                         /lib64/libkrb5.so.3.3 (deleted)
30b06db000-30b06e5000 r--p 000db000 fd:00 263021                         /lib64/libkrb5.so.3.3 (deleted)
30b06e5000-30b06e7000 rw-p 000e5000 fd:00 263021                         /lib64/libkrb5.so.3.3 (deleted)
30b0800000-30b080a000 r-xp 00000000 fd:00 263018                         /lib64/libkrb5support.so.0.1.#prelink#.wtyKWA (deleted)
30b080a000-30b0a09000 ---p 0000a000 fd:00 263018                         /lib64/libkrb5support.so.0.1.#prelink#.wtyKWA (deleted)
30b0a09000-30b0a0a000 r--p 00009000 fd:00 263018                         /lib64/libkrb5support.so.0.1.#prelink#.wtyKWA (deleted)
30b0a0a000-30b0a0b000 rw-p 0000a000 fd:00 263018                         /lib64/libkrb5support.so.0.1.#prelink#.wtyKWA (deleted)
30b0c00000-30b0c29000 r-xp 00000000 fd:00 263019                         /lib64/libk5crypto.so.3.1.#prelink#.cVDFrm (deleted)
30b0c29000-30b0e29000 ---p 00029000 fd:00 263019                         /lib64/libk5crypto.so.3.1.#prelink#.cVDFrm (deleted)
30b0e29000-30b0e2a000 r--p 00029000 fd:00 263019                         /lib64/libk5crypto.so.3.1.#prelink#.cVDFrm (deleted)
30b0e2a000-30b0e2b000 rw-p 0002a000 fd:00 263019                         /lib64/libk5crypto.so.3.1.#prelink#.cVDFrm (deleted)
30b0e2b000-30b0e2c000 rw-p 00000000 00:00 0
30b1000000-30b1002000 r-xp 00000000 fd:00 263017                         /lib64/libkeyutils.so.1.3.#prelink#.AZxisP (deleted)
30b1002000-30b1201000 ---p 00002000 fd:00 263017                         /lib64/libkeyutils.so.1.3.#prelink#.AZxisP (deleted)
30b1201000-30b1202000 r--p 00001000 fd:00 263017                         /lib64/libkeyutils.so.1.3.#prelink#.AZxisP (deleted)
30b1202000-30b1203000 rw-p 00002000 fd:00 263017                         /lib64/libkeyutils.so.1.3.#prelink#.AZxisP (deleted)
30b1400000-30b144e000 r-xp 00000000 fd:00 263016                         /lib64/libldap-2.4.so.2.10.3.#prelink#.k0ZfzL (deleted)
30b144e000-30b164d000 ---p 0004e000 fd:00 263016                         /lib64/libldap-2.4.so.2.10.3.#prelink#.k0ZfzL (deleted)
30b164d000-30b164f000 r--p 0004d000 fd:00 263016                         /lib64/libldap-2.4.so.2.10.3.#prelink#.k0ZfzL (deleted)
30b164f000-30b1651000 rw-p 0004f000 fd:00 263016                         /lib64/libldap-2.4.so.2.10.3.#prelink#.k0ZfzL (deleted)
30b1800000-30b1841000 r-xp 00000000 fd:00 263022                         /lib64/libgssapi_krb5.so.2.2.#prelink#.qYvH1E (deleted)
30b1841000-30b1a41000 ---p 00041000 fd:00 263022                         /lib64/libgssapi_krb5.so.2.2.#prelink#.qYvH1E (deleted)
30b1a41000-30b1a42000 r--p 00041000 fd:00 263022                         /lib64/libgssapi_krb5.so.2.2.#prelink#.qYvH1E (deleted)
30b1a42000-30b1a44000 rw-p 00042000 fd:00 263022                         /lib64/libgssapi_krb5.so.2.2.#prelink#.qYvH1E (deleted)
30b2400000-30b25ba000 r-xp 00000000 fd:00 664067                         /usr/lib64/libcrypto.so.1.0.1e.#prelink#.YjHQlH (deleted)
30b25ba000-30b27ba000 ---p 001ba000 fd:00 664067                         /usr/lib64/libcrypto.so.1.0.1e.#prelink#.YjHQlH (deleted)
30b27ba000-30b27d5000 r--p 001ba000 fd:00 664067                         /usr/lib64/libcrypto.so.1.0.1e.#prelink#.YjHQlH (deleted)
30b27d5000-30b27e1000 rw-p 001d5000 fd:00 664067                         /usr/lib64/libcrypto.so.1.0.1e.#prelink#.YjHQlH (deleted)
30b27e1000-30b27e5000 rw-p 00000000 00:00 0
30b4400000-30b4462000 r-xp 00000000 fd:00 667811                         /usr/lib64/libssl.so.1.0.1e.#prelink#.MTyp7S (deleted)
30b4462000-30b4662000 ---p 00062000 fd:00 667811                         /usr/lib64/libssl.so.1.0.1e.#prelink#.MTyp7S (deleted)
30b4662000-30b4666000 r--p 00062000 fd:00 667811                         /usr/lib64/libssl.so.1.0.1e.#prelink#.MTyp7S (deleted)
30b4666000-30b466c000 rw-p 00066000 fd:00 667811                         /usr/lib64/libssl.so.1.0.1e.#prelink#.MTyp7S (deleted)
3857a00000-3857a16000 r-xp 00000000 fd:00 263069                         /lib64/libgcc_s-4.4.7-20120601.so.1
3857a16000-3857c15000 ---p 00016000 fd:00 263069                         /lib64/libgcc_s-4.4.7-20120601.so.1
3857c15000-3857c16000 rw-p 00015000 fd:00 263069                         /lib64/libgcc_s-4.4.7-20120601.so.1
7f2f68000000-7f2f68021000 rw-p 00000000 00:00 0
7f2f68021000-7f2f6c000000 ---p 00000000 00:00 0
7f2f6debf000-7f2f6dee8000 r-xp 00000000 fd:00 684796                     /usr/pgsql-9.6/lib/plpgsql.so
7f2f6dee8000-7f2f6e0e7000 ---p 00029000 fd:00 684796                     /usr/pgsql-9.6/lib/plpgsql.so
7f2f6e0e7000-7f2f6e0ea000 rw-p 00028000 fd:00 684796                     /usr/pgsql-9.6/lib/plpgsql.so
7f2f6e0ea000-7f2f6e16b000 rw-p 00000000 00:00 0
7f2f6e18c000-7f2f6e18d000 r-xp 00000000 fd:00 684809                     /usr/pgsql-9.6/lib/utf8_and_iso8859_1.so
7f2f6e18d000-7f2f6e38c000 ---p 00001000 fd:00 684809                     /usr/pgsql-9.6/lib/utf8_and_iso8859_1.so
7f2f6e38c000-7f2f6e38d000 rw-p 00000000 fd:00 684809                     /usr/pgsql-9.6/lib/utf8_and_iso8859_1.so
7f2f6e38d000-7f2f6e5ce000 rw-p 00000000 00:00 0
7f2f6e5ce000-7f2f6e5db000 r-xp 00000000 fd:00 262182                     /lib64/libnss_files-2.12.so (deleted)
7f2f6e5db000-7f2f6e7da000 ---p 0000d000 fd:00 262182                     /lib64/libnss_files-2.12.so (deleted)
7f2f6e7da000-7f2f6e7db000 r--p 0000c000 fd:00 262182                     /lib64/libnss_files-2.12.so (deleted)
7f2f6e7db000-7f2f6e7dc000 rw-p 0000d000 fd:00 262182                     /lib64/libnss_files-2.12.so (deleted)
7f2f6e7e1000-7f2f6e7e2000 rw-p 00000000 00:00 0
7f2f6e7e2000-7f2f6e7e9000 r--s 00000000 fd:00 794259                     /usr/lib64/gconv/gconv-modules.cache
7f2f6e7e9000-7f3006c25000 rw-s 00000000 00:04 306667661                  /dev/zero (deleted)
7f3006c25000-7f3006c2d000 r-xp 00000000 fd:00 684965                     /usr/pgsql-9.6/lib/pg_stat_statements.so
7f3006c2d000-7f3006e2c000 ---p 00008000 fd:00 684965                     /usr/pgsql-9.6/lib/pg_stat_statements.so
7f3006e2c000-7f3006e2d000 rw-p 00007000 fd:00 684965                     /usr/pgsql-9.6/lib/pg_stat_statements.so
7f3006e2d000-7f300ccc2000 r--p 00000000 fd:00 684156                     /usr/lib/locale/locale-archive (deleted)
7f300ccc2000-7f300ccd0000 rw-p 00000000 00:00 0
7f300ccd0000-7f300cd53000 r-xp 00000000 fd:00 262257                     /lib64/libm-2.12.so (deleted)
7f300cd53000-7f300cf52000 ---p 00083000 fd:00 262257                     /lib64/libm-2.12.so (deleted)
7f300cf52000-7f300cf53000 r--p 00082000 fd:00 262257                     /lib64/libm-2.12.so (deleted)
7f300cf53000-7f300cf54000 rw-p 00083000 fd:00 262257                     /lib64/libm-2.12.so (deleted)
7f300cf54000-7f300cf57000 rw-p 00000000 00:00 0
7f300cf57000-7f300cf63000 rw-s 00000000 00:10 306667664                  /dev/shm/PostgreSQL.1024409285
7f300cf63000-7f300cf64000 rw-s 00000000 00:04 18907136                   /SYSV0052e2c1 (deleted)
7f300cf64000-7f300cf65000 rw-p 00000000 00:00 0
7ffefba0c000-7ffefba21000 rw-p 00000000 00:00 0                          [stack]
7ffefbada000-7ffefbadb000 r-xp 00000000 00:00 0                          [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
2018-03-12 10:37:03.412 CET - @: LOG:  server process (PID 2750) was terminated by signal 6: Aborted
2018-03-12 10:37:03.412 CET - @: DETAIL:  Failed process was running: WITH RECURSIVE sub2 AS ( WITH RECURSIVE sub AS ( SELECT id, order_entry_fk FROM order_entry WHERE (order_entry.ORDER_STATUS_FK <> 'CA' AND (order_entry.effective_when_begin < Cast( To_Timestamp('2018-03-12 23:59:59.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) AND order_entry.effective_when_end >= Cast( To_Timestamp('2018-03-12 00:00:00.000','YYYY-MM-DD HH24:MI:SS.FF') AS TIMESTAMP) )) UNION ALL SELECT parent.id, parent.order_entry_fk FROM order_entry AS parent JOIN sub AS child ON (parent.id = child.order_entry_fk) ) SELECT id, 1 AS level, CAST(id AS varchar) AS levelpath FROM order_entry WHERE order_entry_fk IS NULL AND id IN (SELECT id FROM sub WHERE sub.order_entry_fk IS NULL  )UNION ALL SELECT child.id, parent.level + 1 AS level, parent.levelpath || '.' || child.id AS levelpath FROM order_entry AS child JOIN sub2 AS parent ON  (child.order_entry_fk = parent.id) )SELECT (SELECT Count(*) FROM order_additional_info WHERE order_entry_fk=order_entry.id) AS order_additional_info_cnt, (SELECT Count(*) FROM or
2018-03-12 10:37:03.412 CET - @: LOG:  terminating any other active server processes