Re: [HACKERS] Deadlock in XLogInsert at AIX

One more assertion failure:


ExceptionalCondition(conditionName = "!(OldPageRqstPtr <= XLogCtl->InitializedUpTo)", errorType = "FailedAssertion", fileName = "xlog.c", lineNumber = 1887), line 54 in "assert.c"

(dbx) p OldPageRqstPtr
153551667200
(dbx) p XLogCtl->InitializedUpTo
153551667200
(dbx) p InitializedUpTo
153551659008

I slightly modify xlog.c code - store value of XLogCtl->InitializedUpTo in local variable:


 1870         LWLockAcquire(WALBufMappingLock, LW_EXCLUSIVE);
 1871
 1872         /*
 1873          * Now that we have the lock, check if someone initialized the page
 1874          * already.
 1875          */
 1876         while (upto >= XLogCtl->InitializedUpTo || opportunistic)
 1877         {
 1878                 XLogRecPtr InitializedUpTo = XLogCtl->InitializedUpTo;
 1879                 nextidx = XLogRecPtrToBufIdx(InitializedUpTo);
 1880
 1881                 /*
 1882                  * Get ending-offset of the buffer page we need to replace (this may
 1883                  * be zero if the buffer hasn't been used yet).  Fall through if it's
 1884                  * already written out.
 1885                  */
 1886                 OldPageRqstPtr = XLogCtl->xlblocks[nextidx];
 1887                 Assert(OldPageRqstPtr <= XLogCtl->InitializedUpTo);


And, as you can see,  XLogCtl->InitializedUpTo is not equal to saved value InitializedUpTo.
But we are under exclusive WALBufMappingLock and InitializedUpTo is updated only under this lock.
So it means that LW-locks doesn't work!
I inspected code of pg_atomic_compare_exchange_u32_impl and didn't sync in prologue:

(dbx) listi pg_atomic_compare_exchange_u32_impl
0x1000817bc (pg_atomic_compare_exchange_u32_impl+0x1c)  e88100b0             ld   r4,0xb0(r1)
0x1000817c0 (pg_atomic_compare_exchange_u32_impl+0x20)  e86100b8             ld   r3,0xb8(r1)
0x1000817c4 (pg_atomic_compare_exchange_u32_impl+0x24)  800100c0            lwz   r0,0xc0(r1)
0x1000817c8 (pg_atomic_compare_exchange_u32_impl+0x28)  7c0007b4          extsw   r0,r0
0x1000817cc (pg_atomic_compare_exchange_u32_impl+0x2c)  e8a30002            lwa   r5,0x0(r3)
0x1000817d0 (pg_atomic_compare_exchange_u32_impl+0x30)  7cc02028          lwarx   r6,r0,r4,0x0
0x1000817d4 (pg_atomic_compare_exchange_u32_impl+0x34)  7c053040           cmpl   cr0,0x0,r5,r6
0x1000817d8 (pg_atomic_compare_exchange_u32_impl+0x38)  4082000c            bne   0x1000817e4 (pg_atomic_compare_exchange_u32_impl+0x44)
0x1000817dc (pg_atomic_compare_exchange_u32_impl+0x3c)  7c00212d         stwcx.   r0,r0,r4
0x1000817e0 (pg_atomic_compare_exchange_u32_impl+0x40)  40e2fff0           bne+   0x1000817d0 (pg_atomic_compare_exchange_u32_impl+0x30)
0x1000817e4 (pg_atomic_compare_exchange_u32_impl+0x44)  60c00000            ori   r0,r6,0x0
0x1000817e8 (pg_atomic_compare_exchange_u32_impl+0x48)  90030000            stw   r0,0x0(r3)
0x1000817ec (pg_atomic_compare_exchange_u32_impl+0x4c)  7c000026           mfcr   r0
0x1000817f0 (pg_atomic_compare_exchange_u32_impl+0x50)  54001ffe         rlwinm   r0,r0,0x3,0x1f,0x1f
0x1000817f4 (pg_atomic_compare_exchange_u32_impl+0x54)  78000620         rldicl   r0,r0,0x0,0x19
0x1000817f8 (pg_atomic_compare_exchange_u32_impl+0x58)  98010070            stb   r0,0x70(r1)
0x1000817fc (pg_atomic_compare_exchange_u32_impl+0x5c)  4c00012c          isync
0x100081800 (pg_atomic_compare_exchange_u32_impl+0x60)  88610070            lbz   r3,0x70(r1)
0x100081804 (pg_atomic_compare_exchange_u32_impl+0x64)  48000004              b   0x100081808 (pg_atomic_compare_exchange_u32_impl+0x68)
0x100081808 (pg_atomic_compare_exchange_u32_impl+0x68)  38210080           addi   r1,0x80(r1)
0x10008180c (pg_atomic_compare_exchange_u32_impl+0x6c)  4e800020            blr


Source code of pg_atomic_compare_exchange_u32_impl is the following:

static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
                                    uint32 *expected, uint32 newval)
{
    bool        ret;

    /*
     * atomics.h specifies sequential consistency ("full barrier semantics")
     * for this interface.  Since "lwsync" provides acquire/release
     * consistency only, do not use it here.  GCC atomics observe the same
     * restriction; see its rs6000_pre_atomic_barrier().
     */
    __asm__ __volatile__ ("    sync \n" ::: "memory");

    /*
     * XXX: __compare_and_swap is defined to take signed parameters, but that
     * shouldn't matter since we don't perform any arithmetic operations.
     */
    ret = __compare_and_swap((volatile int*)&ptr->value,
                             (int *)expected, (int)newval);

    /*
     * xlc's documentation tells us:
     * "If __compare_and_swap is used as a locking primitive, insert a call to
     * the __isync built-in function at the start of any critical sections."
     *
     * The critical section begins immediately after __compare_and_swap().
     */
    __isync();

    return ret;
}

and if I compile this fuctions standalone, I get the following assembler code:

.pg_atomic_compare_exchange_u32_impl:   # 0x0000000000000000 (H.4.NO_SYMBOL)
        stdu       SP,-128(SP)
        std        r3,176(SP)
        std        r4,184(SP)
        std        r5,192(SP)
        ld         r0,192(SP)
        stw        r0,192(SP)
        sync      
        ld         r4,176(SP)
        ld         r3,184(SP)
        lwz        r0,192(SP)
        extsw      r0,r0
        lwa        r5,0(r3)
__L30:                                  # 0x0000000000000030 (H.4.NO_SYMBOL+0x030)
        lwarx      r6,r0,r4
        cmpl       0,0,r5,r6
        bc         BO_IF_NOT,CR0_EQ,__L44
        stwcx.     r0,r0,r4
        .machine        "any"
        bc         BO_IF_NOT_3,CR0_EQ,__L30
__L44:                                  # 0x0000000000000044 (H.4.NO_SYMBOL+0x044)
        ori        r0,r6,0x0000
        stw        r0,0(r3)
        mfcr       r0
        rlwinm     r0,r0,3,31,31
        rldicl     r0,r0,0,56
        stb        r0,112(SP)
        isync     
        lbz        r3,112(SP)
        addi       SP,SP,128
        bclr       BO_ALWAYS,CR0_LT

sync is here!

-- 
Konstantin Knizhnik
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company