extend recycler verify mark to verify missing write barrier

record every write barrier bit set, while verifying mark, if an object is marked, verify the referencing pointer DID set the write barrier bit.

bin/rl/rlrun.cpp

@@ -485,7 +485,7 @@ int
     }
     else if (kind == TK_JSCRIPT || kind==TK_HTML || kind == TK_COMMAND)
     {
-        char tempExtraCCFlags[MAX_PATH] = {0};
+        char tempExtraCCFlags[MAX_PATH*2] = {0};
 
         // Only append when EXTRA_CC_FLAGS isn't empty.
         if (EXTRA_CC_FLAGS[0])

lib/Common/Common/Jobs.cpp

@@ -26,6 +26,7 @@
 #include "Common/Jobs.h"
 #include "Common/Jobs.inl"
 #include "Core/CommonMinMax.h"
+#include "Memory/RecyclerWriteBarrierManager.h"
 
 namespace JsUtil
 {
@@ -1224,6 +1225,11 @@ namespace JsUtil
     unsigned int WINAPI BackgroundJobProcessor::StaticThreadProc(void *lpParam)
     {
         Assert(lpParam);
+
+#ifdef _M_X64_OR_ARM64
+        Memory::RecyclerWriteBarrierManager::OnThreadInit();
+#endif
+
 #if !defined(_UCRT)
         HMODULE dllHandle = NULL;
         if (!GetModuleHandleEx(0, AutoSystemInfo::GetJscriptDllFileName(), &dllHandle))

lib/Common/Memory/HeapBlock.cpp

@@ -813,7 +813,7 @@ SmallHeapBlockT<TBlockAttributes>::VerifyMark()
 
             if (!this->IsLeafBlock()
 #ifdef RECYCLER_WRITE_BARRIER
-                && !this->IsWithBarrier()
+                && (!this->IsWithBarrier() || CONFIG_FLAG(ForceSoftwareWriteBarrier))
 #endif
                 )
             {
@@ -823,7 +823,10 @@ SmallHeapBlockT<TBlockAttributes>::VerifyMark()
                     for (uint i = 0; i < objectWordCount; i++)
                     {
                         void* target = *(void**) objectAddress;
-                        recycler->VerifyMark(target);
+                        if (recycler->VerifyMark(target))
+                        {
+                            Assert(this->wbVerifyBits.Test((BVIndex)(objectAddress - this->address) / sizeof(void*)));
+                        }
 
                         objectAddress += sizeof(void *);
                     }
@@ -835,7 +838,7 @@ SmallHeapBlockT<TBlockAttributes>::VerifyMark()
 }
 
 template <class TBlockAttributes>
-void
+bool
 SmallHeapBlockT<TBlockAttributes>::VerifyMark(void * objectAddress)
 {
     // Because we mark through new object, we might have a false reference
@@ -844,19 +847,20 @@ SmallHeapBlockT<TBlockAttributes>::VerifyMark(void * objectAddress)
     // Can't verify when the block is new
     if (this->heapBucket->GetRecycler()->heapBlockMap.IsAddressInNewChunk(objectAddress))
     {
-        return;
+        return false;
     }
 
     ushort bitIndex = GetAddressBitIndex(objectAddress);
-
+    bool isMarked = this->GetMarkedBitVector()->Test(bitIndex) == TRUE;
 #if DBG
-    Assert(this->GetMarkedBitVector()->Test(bitIndex));
+    Assert(isMarked);
 #else
-    if (!this->GetMarkedBitVector()->Test(bitIndex))
+    if (!isMarked)
     {
         DebugBreak();
     }
 #endif
+    return isMarked;
 }
 
 #endif
@@ -1262,6 +1266,13 @@ SmallHeapBlockT<TBlockAttributes>::EnqueueProcessedObject(FreeObject ** list, vo
     freeObject->SetNext(*list);
     *list = freeObject;
 
+#if DBG
+    if (CONFIG_FLAG(ForceSoftwareWriteBarrier))
+    {
+        this->WBClearBits((char*)objectAddress);
+    }
+#endif
+
     // clear the attributes so that when we are allocating a leaf, we don't have to set the attribute
     this->ObjectInfo(index) = 0;
 }

lib/Common/Memory/HeapBlock.h

@@ -315,8 +315,11 @@ public:
 
         return false;
     }
-
 #if DBG
+    virtual void WBSetBit(char* addr) = 0;
+    virtual void WBSetBits(char* addr, uint length) = 0;
+    virtual void WBClearBits(char* addr) = 0;
+
     virtual BOOL IsFreeObject(void* objectAddress) = 0;
 #endif
     virtual BOOL IsValidObject(void* objectAddress) = 0;
@@ -328,7 +331,7 @@ public:
     virtual void SetObjectMarkedBit(void* objectAddress) = 0;
 
 #ifdef RECYCLER_VERIFY_MARK
-    virtual void VerifyMark(void * objectAddress) = 0;
+    virtual bool VerifyMark(void * objectAddress) = 0;
 #endif
 #ifdef PROFILE_RECYCLER_ALLOC
     virtual void * GetTrackerData(void * address) = 0;
@@ -433,6 +436,9 @@ public:
 #endif
     SmallHeapBlockBitVector* markBits;
     SmallHeapBlockBitVector  freeBits;
+#if DBG
+    BVStatic<TBlockAttributes::PageCount * AutoSystemInfo::PageSize / sizeof(void*)> wbVerifyBits;
+#endif
 
 #if DBG || defined(RECYCLER_STATS)
     SmallHeapBlockBitVector debugFreeBits;
@@ -452,6 +458,24 @@ public:
     void ProtectUnusablePages() {}
     void RestoreUnusablePages() {}
 
+#if DBG
+    virtual void WBSetBit(char* addr) override
+    {
+        uint index = (uint)(addr - this->address) / sizeof(void*);
+        wbVerifyBits.Set(index);
+    }
+    virtual void WBSetBits(char* addr, uint length) override
+    {
+        uint index = (uint)(addr - this->address) / sizeof(void*);
+        wbVerifyBits.SetRange(index, length);
+    }
+    virtual void WBClearBits(char* addr) override
+    {
+        uint index = (uint)(addr - this->address) / sizeof(void*);
+        wbVerifyBits.ClearRange(index, this->objectSize / sizeof(void*));
+    }
+#endif
+
     uint GetUnusablePageCount()
     {
         return 0;
@@ -602,7 +626,7 @@ public:
 #endif
 #ifdef RECYCLER_VERIFY_MARK
     void VerifyMark();
-    virtual void VerifyMark(void * objectAddress) override;
+    virtual bool VerifyMark(void * objectAddress) override;
 #endif
 #ifdef RECYCLER_PERF_COUNTERS
     virtual void UpdatePerfCountersOnFree() override sealed;

lib/Common/Memory/LargeHeapBlock.cpp

@@ -175,6 +175,9 @@ LargeHeapBlock::LargeHeapBlock(__in char * address, size_t pageCount, Segment *
 #if defined(RECYCLER_PAGE_HEAP) && defined(STACK_BACK_TRACE)
     , pageHeapAllocStack(nullptr), pageHeapFreeStack(nullptr)
 #endif
+#if DBG
+    ,wbVerifyBits(&HeapAllocator::Instance)
+#endif
 {
     Assert(address != nullptr);
     Assert(pageCount != 0);
@@ -846,21 +849,24 @@ LargeHeapBlock::VerifyMark()
         {
             void* target = *(void **)objectAddress;
 
-            recycler->VerifyMark(target);
+            if (recycler->VerifyMark(target))
+            {
+                Assert(this->wbVerifyBits.Test((BVIndex)(objectAddress - this->address) / sizeof(void*)));
+            }
 
             objectAddress += sizeof(void *);
         }
     }
 }
 
-void
+bool
 LargeHeapBlock::VerifyMark(void * objectAddress)
 {
     LargeObjectHeader * header = GetHeader(objectAddress);
 
     if ((char *)header < this->address)
     {
-        return;
+        return false;
     }
 
     uint index = header->objectIndex;
@@ -868,13 +874,13 @@ LargeHeapBlock::VerifyMark(void * objectAddress)
     if (index >= this->allocCount)
     {
         // object not allocated
-        return;
+        return false;
     }
 
     if (this->HeaderList()[index] != header)
     {
         // header doesn't match, not a real object
-        return;
+        return false;
     }
 
     bool isMarked = this->heapInfo->recycler->heapBlockMap.IsMarked(objectAddress);
@@ -887,6 +893,7 @@ LargeHeapBlock::VerifyMark(void * objectAddress)
         DebugBreak();
     }
 #endif
+    return isMarked;
 }
 
 #endif
@@ -2082,3 +2089,42 @@ LargeHeapBlock::CapturePageHeapFreeStack()
 #endif
 }
 #endif
+
+#if DBG
+void LargeHeapBlock::WBSetBit(char* addr)
+{
+    uint index = (uint)(addr - this->address) / sizeof(void*);
+    try
+    {
+        AUTO_NESTED_HANDLED_EXCEPTION_TYPE(static_cast<ExceptionType>(ExceptionType_DisableCheck));
+        wbVerifyBits.Set(index);
+    }
+    catch (Js::OutOfMemoryException&)
+    {
+    }
+}
+void LargeHeapBlock::WBSetBits(char* addr, uint length)
+{
+    uint index = (uint)(addr - this->address) / sizeof(void*);
+    try
+    {
+        AUTO_NESTED_HANDLED_EXCEPTION_TYPE(static_cast<ExceptionType>(ExceptionType_DisableCheck));
+        for (uint i = 0; i < length; i++)
+        {
+            wbVerifyBits.Set(index + i);
+        }
+    }
+    catch (Js::OutOfMemoryException&)
+    {
+    }
+}
+void LargeHeapBlock::WBClearBits(char* addr)
+{
+    uint index = (uint)(addr - this->address) / sizeof(void*);
+    size_t objectSize = this->GetHeader(addr)->objectSize;
+    for (uint i = 0; i < (uint)objectSize / sizeof(void*); i++)
+    {
+        wbVerifyBits.Clear(index + i);
+    }
+}
+#endif

lib/Common/Memory/LargeHeapBlock.h

@@ -171,7 +171,7 @@ public:
 #endif
 #ifdef RECYCLER_VERIFY_MARK
     void VerifyMark();
-    virtual void VerifyMark(void * objectAddress) override;
+    virtual bool VerifyMark(void * objectAddress) override;
 #endif
 #ifdef RECYCLER_PERF_COUNTERS
     virtual void UpdatePerfCountersOnFree() override;
@@ -302,5 +302,14 @@ public:
 #ifdef PROFILE_RECYCLER_ALLOC
     void ** GetTrackerDataArray();
 #endif
+
+#if DBG
+private:
+    BVSparse<HeapAllocator> wbVerifyBits;
+public:
+    virtual void WBSetBit(char* addr) override;
+    virtual void WBSetBits(char* addr, uint length) override;
+    virtual void WBClearBits(char* addr) override;
+#endif
 };
 }

lib/Common/Memory/Recycler.cpp

@@ -460,6 +460,24 @@ Recycler::~Recycler()
 #if ENABLE_CONCURRENT_GC
     Assert(!this->isAborting);
 #endif
+#if DBG
+    if (CONFIG_FLAG(ForceSoftwareWriteBarrier))
+    {
+        if (recyclerList == this)
+        {
+            recyclerList = this->next;
+        }
+        else
+        {
+            Recycler* list = recyclerList;
+            while (list->next != this)
+            {
+                list = list->next;
+            }
+            list->next = this->next;
+        }
+    }
+#endif
 
     // Stop any further collection
     this->isShuttingDown = true;
@@ -716,6 +734,9 @@ Recycler::RootRelease(void* obj, uint *count)
     // another GC if there is an exhaustive GC going on.
     this->CollectNow<CollectExhaustiveCandidate>();
 }
+#if DBG
+Recycler* Recycler::recyclerList = nullptr;
+#endif
 
 void
 Recycler::Initialize(const bool forceInThread, JsUtil::ThreadService *threadService, const bool deferThreadStartup
@@ -900,6 +921,13 @@ Recycler::Initialize(const bool forceInThread, JsUtil::ThreadService *threadServ
 #else
     Assert(!needWriteWatch);
 #endif
+#if DBG
+    if (CONFIG_FLAG(ForceSoftwareWriteBarrier))
+    {
+        this->next = recyclerList;
+        recyclerList = this;
+    }
+#endif
 }
 
 BOOL
@@ -7568,9 +7596,7 @@ Recycler::TrackAllocCore(void * object, size_t size, const TrackAllocData& track
     
     if (!trackerDictionary->TryGetValue(typeInfo, &item))
     {
-        if (CONFIG_FLAG(KeepRecyclerTrackData) && 
-            (typeInfo == &typeid(void*)
-                || typeInfo == &typeid(int*))) // type info is not useful record stack instead
+        if (CONFIG_FLAG(KeepRecyclerTrackData) && isArray) // type info is not useful record stack instead
         {
             size_t stackTraceSize = 16 * sizeof(void*);
             item = NoCheckHeapNewPlus(stackTraceSize, TrackerItem, typeInfo);
@@ -7951,7 +7977,7 @@ Recycler::VerifyMarkStack()
     }
 }
 
-void
+bool
 Recycler::VerifyMark(void * candidate)
 {
     void * realAddress;
@@ -7961,12 +7987,12 @@ Recycler::VerifyMark(void * candidate)
         heapBlock = heapBlockMap.GetHeapBlock(candidate);
         if (heapBlock == nullptr)
         {
-            return;
+            return false;
         }
         realAddress = heapBlock->GetRealAddressFromInterior(candidate);
         if (realAddress == nullptr)
         {
-            return;
+            return false;
         }
     }
     else
@@ -7974,11 +8000,11 @@ Recycler::VerifyMark(void * candidate)
         heapBlock = this->FindHeapBlock(candidate);
         if (heapBlock == nullptr)
         {
-            return;
+            return false;
         }
         realAddress = candidate;
     }
-    heapBlock->VerifyMark(realAddress);
+    return heapBlock->VerifyMark(realAddress);
 }
 #endif
 
@@ -8491,6 +8517,38 @@ Recycler::NotifyFree(__in char *address, size_t size)
 #endif
 }
 
+#if DBG
+void 
+Recycler::WBSetBit(char* addr)
+{
+    Recycler* recycler = Recycler::recyclerList;
+    while (recycler)
+    {
+        auto heapBlock = recycler->FindHeapBlock((void*)((UINT_PTR)addr&~HeapInfo::ObjectAlignmentMask));
+        if (heapBlock)
+        {
+            heapBlock->WBSetBit(addr);
+            break;
+        }
+        recycler = recycler->next;
+    }
+}
+void
+Recycler::WBSetBits(char* addr, uint length)
+{
+    Recycler* recycler = Recycler::recyclerList;
+    while (recycler)
+    {
+        auto heapBlock = recycler->FindHeapBlock((void*)((UINT_PTR)addr&~HeapInfo::ObjectAlignmentMask));
+        if (heapBlock)
+        {
+            heapBlock->WBSetBits(addr, length);
+        }
+        recycler = recycler->next;
+    }
+}
+#endif
+
 size_t
 RecyclerHeapObjectInfo::GetSize() const
 {

lib/Common/Memory/Recycler.h

@@ -1696,7 +1696,7 @@ private:
     void VerifyMarkArena(ArenaData * arena);
     void VerifyMarkBigBlockList(BigBlock * memoryBlocks);
     void VerifyMarkArenaMemoryBlockList(ArenaMemoryBlock * memoryBlocks);
-    void VerifyMark(void * address);
+    bool VerifyMark(void * address);
 #endif
 #if DBG_DUMP
     bool forceTraceMark;
@@ -1921,6 +1921,15 @@ private:
     } objectBeforeCollectCallbackState;
 
     bool ProcessObjectBeforeCollectCallbacks(bool atShutdown = false);
+
+#if DBG
+private:
+    static Recycler* recyclerList;
+    Recycler* next;
+public:
+    static void WBSetBit(char* addr);
+    static void WBSetBits(char* addr, uint length);
+#endif
 };
 
 
@@ -2092,6 +2101,10 @@ class CollectedRecyclerWeakRefHeapBlock : public HeapBlock
 {
 public:
 #if DBG
+    virtual void WBSetBit(char* addr) override { Assert(false); }
+    virtual void WBSetBits(char* addr, uint length) override { Assert(false); }
+    virtual void WBClearBits(char* addr) override { Assert(false); }
+
     virtual BOOL IsFreeObject(void* objectAddress) override { Assert(false); return false; }
 #endif
     virtual BOOL IsValidObject(void* objectAddress) override { Assert(false); return false; }
@@ -2102,7 +2115,7 @@ public:
     virtual void SetObjectMarkedBit(void* objectAddress) override { Assert(false); }
 
 #ifdef RECYCLER_VERIFY_MARK
-    virtual void VerifyMark(void * objectAddress) override { Assert(false); }
+    virtual bool VerifyMark(void * objectAddress) override { Assert(false); return false; }
 #endif
 #ifdef RECYCLER_PERF_COUNTERS
     virtual void UpdatePerfCountersOnFree() override { Assert(false); }

lib/Common/Memory/Recycler.inl

@@ -265,6 +265,14 @@ Recycler::AllocZeroWithAttributesInlined(DECLSPEC_GUARD_OVERFLOW size_t size)
 #if DBG
     VerifyPageHeapFillAfterAlloc<attributes>(obj, size);
 #endif
+
+#if DBG
+    if (CONFIG_FLAG(ForceSoftwareWriteBarrier))
+    {
+        this->FindHeapBlock(obj)->WBClearBits(obj);
+    }
+#endif
+
     return obj;
 }
 

lib/Common/Memory/RecyclerWriteBarrierManager.cpp

@@ -310,6 +310,9 @@ RecyclerWriteBarrierManager::WriteBarrier(void * address)
         Output::Print(_u("Writing to 0x%p (CIndex: %u)\n"), address, index);
     }
 #endif
+#if DBG
+    Recycler::WBSetBit((char*)address);
+#endif
 }
 
 void
@@ -325,6 +328,11 @@ RecyclerWriteBarrierManager::WriteBarrier(void * address, size_t bytes)
     Assert(startIndex <= endIndex);
     memset(cardTable + startIndex, WRITE_BARRIER_PAGE_BIT | DIRTYBIT, endIndex - startIndex);
     GlobalSwbVerboseTrace(_u("Writing to 0x%p (CIndex: %u-%u)\n"), address, startIndex, endIndex);
+
+#if DBG
+    Recycler::WBSetBits((char*)address, (uint)bytes/sizeof(void*));
+#endif
+
 #else
     uint bitShift = (((uint)address) >> s_BitArrayCardTableShift);
     uint bitMask = 0xFFFFFFFF << bitShift;