diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 6ac07f2fda..2ec5158866 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2069,6 +2069,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 	bool		all_visible_cleared = false;
 	bool		all_frozen_set = false;
 	uint8		vmstatus = 0;
+	XLogRecPtr	recptr = InvalidXLogRecPtr;
 
 	/* Cheap, simplistic check that the tuple matches the rel's rowtype. */
 	Assert(HeapTupleHeaderGetNatts(tup->t_data) <=
@@ -2179,7 +2180,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 	{
 		xl_heap_insert xlrec;
 		xl_heap_header xlhdr;
-		XLogRecPtr	recptr;
 		Page		page = BufferGetPage(buffer);
 		uint8		info = XLOG_HEAP_INSERT;
 		int			bufflags = 0;
@@ -2275,7 +2275,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		 * visibility rules.
 		 */
 		visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
-						  InvalidXLogRecPtr, vmbuffer,
+						  recptr, vmbuffer,
 						  InvalidTransactionId,
 						  VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
 	}
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index e198df65d8..c00544525f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -266,6 +266,10 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf,
 	if (!BufferIsValid(vmBuf) || BufferGetBlockNumber(vmBuf) != mapBlock)
 		elog(ERROR, "wrong VM buffer passed to visibilitymap_set");
 
+	/* bail out if we already got a valid LSN */
+	if (recptr != InvalidXLogRecPtr)
+		return;
+
 	page = BufferGetPage(vmBuf);
 	map = (uint8 *) PageGetContents(page);
 	LockBuffer(vmBuf, BUFFER_LOCK_EXCLUSIVE);