Move 2-pass check to the background thread.

lib/Common/CommonDefines.h

@@ -171,26 +171,27 @@
 #endif
 
 
+#ifndef ENABLE_VALGRIND
+#define ENABLE_CONCURRENT_GC 1
+#ifdef _WIN32
+#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 1 // Needs ENABLE_CONCURRENT_GC to be enabled for this to be enabled.
+#else
+#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 0 // Needs ENABLE_CONCURRENT_GC to be enabled for this to be enabled.
+#endif
+#else
+#define ENABLE_CONCURRENT_GC 0
+#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 0 // Needs ENABLE_CONCURRENT_GC to be enabled for this to be enabled.
+#endif
+
 #ifdef _WIN32
 #define SYSINFO_IMAGE_BASE_AVAILABLE 1
-#define ENABLE_CONCURRENT_GC 1
-#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 1 // Only takes effect when ENABLE_CONCURRENT_GC is enabled.
-#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST 1 // Use Interlocked SLIST for allocableHeapBlockList
 #define SUPPORT_WIN32_SLIST 1
 #ifndef CHAKRACORE_LITE
 #define ENABLE_JS_ETW                               // ETW support
 #endif
 #else
 #define SYSINFO_IMAGE_BASE_AVAILABLE 0
-#ifndef ENABLE_VALGRIND
-#define ENABLE_CONCURRENT_GC 1
-#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 1 // Only takes effect when ENABLE_CONCURRENT_GC is enabled.
-#else
-#define ENABLE_CONCURRENT_GC 0
-#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 0 // Only takes effect when ENABLE_CONCURRENT_GC is enabled.
-#endif
 #define SUPPORT_WIN32_SLIST 0
-#define ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST 0 // Use Interlocked SLIST for allocableHeapBlockList
 #endif
 
 #ifdef CHAKRACORE_LITE

lib/Common/Memory/HeapBlock.h

@@ -413,13 +413,11 @@ public:
         return (heapBlockType);
     }
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
+#if (DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)) && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     bool WasAllocatedFromDuringSweep()
     {
         return this->wasAllocatedFromDuringSweep;
     }
-#endif
 #endif
 
     IdleDecommitPageAllocator* GetPageAllocator(Recycler* recycler);
@@ -472,7 +470,7 @@ public:
 #endif
 };
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
 template <typename TBlockType>
 struct HeapBlockSListItem {
     // SLIST_ENTRY needs to be the first element in the structure to avoid calculating offset with the SList API calls.

lib/Common/Memory/HeapBucket.cpp

@@ -53,7 +53,7 @@ HeapBucketT<TBlockType>::HeapBucketT() :
     fullBlockList(nullptr),
     heapBlockList(nullptr),
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     lastKnownNextAllocableBlockHead(nullptr),
     allocableHeapBlockListHead(nullptr),
     sweepableHeapBlockList(nullptr),
@@ -78,7 +78,7 @@ HeapBucketT<TBlockType>::~HeapBucketT()
     DeleteHeapBlockList(this->fullBlockList);
 
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     if (allocableHeapBlockListHead != nullptr)
     {
         if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
@@ -131,7 +131,7 @@ HeapBucketT<TBlockType>::DeleteHeapBlockList(TBlockType * list)
     DeleteHeapBlockList(list, this->heapInfo->recycler);
 }
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
 template<typename TBlockType>
 bool
 HeapBucketT<TBlockType>::PushHeapBlockToSList(PSLIST_HEADER list, TBlockType * heapBlock)
@@ -395,11 +395,11 @@ HeapBucketT<TBlockType>::HasPendingDisposeHeapBlocks() const
 
 #endif
 
-#if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
 template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::AssertCheckHeapBlockNotInAnyList(TBlockType * heapBlock)
 {
+#if DBG
     AssertMsg(!HeapBlockList::Contains(heapBlock, heapBlockList), "The heap block already exists in the heapBlockList.");
     AssertMsg(!HeapBlockList::Contains(heapBlock, fullBlockList), "The heap block already exists in the fullBlockList.");
     AssertMsg(!HeapBlockList::Contains(heapBlock, emptyBlockList), "The heap block already exists in the emptyBlockList.");
@@ -407,8 +407,10 @@ HeapBucketT<TBlockType>::AssertCheckHeapBlockNotInAnyList(TBlockType * heapBlock
     AssertMsg(!HeapBlockList::Contains(heapBlock, sweepableHeapBlockList), "The heap block already exists in the sweepableHeapBlockList.");
     AssertMsg(!HeapBlockList::Contains(heapBlock, pendingSweepPrepHeapBlockList), "The heap block already exists in the pendingSweepPrepHeapBlockList.");
 #endif
+#endif
 }
 
+#if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
 template <typename TBlockType>
 size_t
 HeapBucketT<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount) const
@@ -419,7 +421,7 @@ HeapBucketT<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount) const
 
 #if ENABLE_CONCURRENT_GC
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         allocatingDuringConcurrentSweep = true;
@@ -446,7 +448,7 @@ HeapBucketT<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount) const
 #endif
 
     // There is no way to determine the number of item in an SLIST if there are >= 65535 items in the list.
-    RECYCLER_SLOW_CHECK(Assert(!checkCount || heapBlockCount == currentHeapBlockCount /*|| (heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)*/));
+    RECYCLER_SLOW_CHECK(Assert(!checkCount || heapBlockCount == currentHeapBlockCount || (heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)));
 
     return currentHeapBlockCount;
 }
@@ -473,7 +475,7 @@ HeapBucketT<TBlockType>::TryAlloc(Recycler * recycler, TBlockAllocatorType * all
 
     TBlockType * heapBlock = this->nextAllocableBlockHead;
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP 
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     bool heapBlockFromAllocableHeapBlockList = false;
     DebugOnly(bool heapBlockInPendingSweepPrepList = false);
 
@@ -489,7 +491,7 @@ HeapBucketT<TBlockType>::TryAlloc(Recycler * recycler, TBlockAllocatorType * all
         if (heapBlock != nullptr)
         {
             Assert(!this->IsAnyFinalizableBucket());
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
             heapBlock->wasAllocatedFromDuringSweep = true;
 #endif
@@ -530,7 +532,7 @@ HeapBucketT<TBlockType>::TryAlloc(Recycler * recycler, TBlockAllocatorType * all
    {
         Assert(!this->IsAllocationStopped());
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         // When allocations are allowed during concurrent sweep we set nextAllocableBlockHead to NULL as the allocator will pick heap blocks from the
         // interlocked SLIST. During that time, the heap block at the top of the SLIST is always the nextAllocableBlockHead.
         // If the heapBlock was just picked from the SLIST and nextAllocableBlockHead is not NULL then we just resumed normal allocations on the background thread
@@ -691,18 +693,6 @@ HeapBucketT<TBlockType>::SnailAlloc(Recycler * recycler, TBlockAllocatorType * a
         }
 #endif
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        // If we started allocations during concurrent sweep; we may have found blocks to allocate from; try again.
-        //if (this->allocationsStartedDuringConcurrentSweep)
-        //{
-        //    memBlock = this->TryAlloc(recycler, allocator, sizeCat, attributes);
-        //    if (memBlock != nullptr)
-        //    {
-        //        return memBlock;
-        //    }
-        //}
-#endif
-
         // We didn't collect, try to add a new heap block
         memBlock = TryAllocFromNewHeapBlock(recycler, allocator, sizeCat, size, attributes);
         if (memBlock != nullptr)
@@ -847,7 +837,7 @@ HeapBucketT<TBlockType>::ResetMarks(ResetMarkFlags flags)
             Assert(!heapBlock->HasFreeObject());
         });
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
         {
             HeapBlockList::ForEach(sweepableHeapBlockList, [flags](TBlockType * heapBlock)
@@ -903,7 +893,7 @@ HeapBucketT<TBlockType>::ScanNewImplicitRoots(Recycler * recycler)
         heapBlock->ScanNewImplicitRoots(recycler);
     });
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(sweepableHeapBlockList, [recycler](TBlockType * heapBlock)
@@ -1109,7 +1099,7 @@ HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlock
             // blocks that have swept object. Queue up the block for concurrent sweep.
             Assert(queuePendingSweep);
             TBlockType *& pendingSweepList = recyclerSweep.GetPendingSweepBlockList(this);
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             AssertMsg(!HeapBlockList::Contains(heapBlock, pendingSweepList), "The heap block already exists in the pendingSweepList.");
 
             heapBlock->SetNextBlock(pendingSweepList);
@@ -1147,7 +1137,7 @@ HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlock
             // finalizable objects, so that we can go through and call the dispose, and then
             // transfer the finalizable object back to the free list.
             SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes> * finalizableHeapBucket = (SmallFinalizableHeapBucketT<typename TBlockType::HeapBlockAttributes>*)this;
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             //AssertMsg(!HeapBlockList::Contains(heapBlock, finalizableHeapBucket->pendingDisposeList), "The heap block already exists in the pendingDisposeList.");
             heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>()->SetNextBlock(finalizableHeapBucket->pendingDisposeList);
             finalizableHeapBucket->pendingDisposeList = heapBlock->template AsFinalizableBlock<typename TBlockType::HeapBlockAttributes>();
@@ -1162,7 +1152,7 @@ HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlock
         {
             Assert(this->nextAllocableBlockHead == nullptr);
             Assert(heapBlock->HasFreeObject());
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
 
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
             if (this->AllocationsStartedDuringConcurrentSweep())
@@ -1205,7 +1195,7 @@ HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlock
         case SweepStateFull:
         {
             Assert(!heapBlock->HasFreeObject());
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             heapBlock->SetNextBlock(this->fullBlockList);
             this->fullBlockList = heapBlock;
 #ifdef RECYCLER_TRACE
@@ -1229,7 +1219,7 @@ HeapBucketT<TBlockType>::SweepHeapBlockList(RecyclerSweep& recyclerSweep, TBlock
 #if ENABLE_CONCURRENT_GC
             // CONCURRENT-TODO: Finalizable block never have background == true and always be processed
             // in thread, so it will not queue up the pages even if we are doing concurrent GC
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
             if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
             {
@@ -1300,7 +1290,7 @@ HeapBucketT<TBlockType>::SweepBucket(RecyclerSweep& recyclerSweep)
     Assert(!recyclerSweep.IsBackground());
 #endif
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && this->sweepableHeapBlockList != nullptr)
     {
         Assert(!this->IsAnyFinalizableBucket());
@@ -1415,7 +1405,7 @@ HeapBucketT<TBlockType>::StopAllocationBeforeSweep()
 {
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     this->allocationsStartedDuringConcurrentSweep = false;
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     this->lastKnownNextAllocableBlockHead = this->nextAllocableBlockHead;
 #endif
 #endif
@@ -1448,7 +1438,7 @@ HeapBucketT<TBlockType>::StartAllocationDuringConcurrentSweep()
     Assert(!this->allocationsStartedDuringConcurrentSweep);
     this->allocationsStartedDuringConcurrentSweep = true;
 
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     // When allocations are allowed during concurrent sweep we set nextAllocableBlockHead to NULL as the allocator will pick heap blocks from the
     // interlocked SLIST. During that time, the heap block at the top of the SLIST is always the nextAllocableBlockHead.
     this->nextAllocableBlockHead = nullptr;
@@ -1502,7 +1492,7 @@ template<typename TBlockType>
 void
 HeapBucketT<TBlockType>::PrepareForAllocationsDuringConcurrentSweep(TBlockType * &currentHeapBlockList)
 {
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     if (this->AllowAllocationsDuringConcurrentSweep())
     {
         this->EnsureAllocableHeapBlockList();
@@ -1533,7 +1523,7 @@ HeapBucketT<TBlockType>::PrepareForAllocationsDuringConcurrentSweep(TBlockType *
                 {
                     // This heap block is NOT ready to be swept concurrently as it hasn't yet been through sweep prep (i.e. Pass1 of sweep).
                     heapBlock->isPendingConcurrentSweepPrep = true;
-                    DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+                    DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                     bool blockAddedToSList = HeapBucketT<TBlockType>::PushHeapBlockToSList(this->allocableHeapBlockListHead, heapBlock);
 
                     // If we encountered OOM while pushing the heapBlock to the SLIST we must add it to the heapBlockList so we don't lose track of it.
@@ -1694,8 +1684,7 @@ HeapBucketT<TBlockType>::FinishConcurrentSweepPass1(RecyclerSweep& recyclerSweep
         TBlockType * heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         while (heapBlock != nullptr)
         {
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
-            AssertMsg(!HeapBlockList::Contains(heapBlock, currentPendingSweepPrepHeapBlockList), "The heap block already exists in the currentPendingSweepPrepHeapBlockList.");
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             if (heapBlock->isPendingConcurrentSweepPrep)
             {
 #ifdef RECYCLER_TRACE
@@ -1750,7 +1739,7 @@ template <typename TBlockType>
 void
 HeapBucketT<TBlockType>::EnsureAllocableHeapBlockList()
 {
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
     {
         if (allocableHeapBlockListHead == nullptr)
@@ -1790,7 +1779,7 @@ HeapBucketT<TBlockType>::FinishConcurrentSweep()
 {
     if (this->AllocationsStartedDuringConcurrentSweep())
     {
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
         Assert(!this->IsAnyFinalizableBucket());
         Assert(this->allocableHeapBlockListHead != nullptr);
 
@@ -1806,7 +1795,7 @@ HeapBucketT<TBlockType>::FinishConcurrentSweep()
         TBlockType * heapBlock = PopHeapBlockFromSList(this->allocableHeapBlockListHead);
         while (heapBlock != nullptr)
         {
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             AssertMsg(!heapBlock->isPendingConcurrentSweepPrep, "The blocks in the SLIST at this time should NOT have sweep prep i.e. sweep-Pass1 pending.");
             newNextAllocableBlockHead = heapBlock;
             heapBlock->SetNextBlock(this->heapBlockList);
@@ -1867,7 +1856,7 @@ HeapBucketT<TBlockType>::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBa
 {
     UpdateAllocators();
     HeapBucket::EnumerateObjects(fullBlockList, infoBits, CallBackFunction);
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBucket::EnumerateObjects(sweepableHeapBlockList, infoBits, CallBackFunction);
@@ -1898,7 +1887,7 @@ HeapBucketT<TBlockType>::Check(bool checkCount)
     UpdateAllocators();
     size_t smallHeapBlockCount = HeapInfo::Check(true, false, this->fullBlockList);
     bool allocatingDuringConcurrentSweep = false;
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         allocatingDuringConcurrentSweep = true;
@@ -1913,7 +1902,7 @@ HeapBucketT<TBlockType>::Check(bool checkCount)
 #endif
     smallHeapBlockCount += HeapInfo::Check(true, false, this->heapBlockList, this->nextAllocableBlockHead);
     smallHeapBlockCount += HeapInfo::Check(false, false, this->nextAllocableBlockHead);
-    Assert(!checkCount || this->heapBlockCount == smallHeapBlockCount /*|| (this->heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)*/);
+    Assert(!checkCount || this->heapBlockCount == smallHeapBlockCount || (this->heapBlockCount >= 65535 && allocatingDuringConcurrentSweep));
     return smallHeapBlockCount;
 }
 #endif
@@ -1943,7 +1932,7 @@ HeapBucketT<TBlockType>::AggregateBucketStats()
     };
 
     HeapBlockList::ForEach(fullBlockList, blockStatsAggregator);
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(sweepableHeapBlockList, blockStatsAggregator);
@@ -1983,7 +1972,7 @@ HeapBucketT<TBlockType>::Verify()
         heapBlock->Verify();
     });
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
 #if DBG
@@ -2079,7 +2068,7 @@ HeapBucketT<TBlockType>::VerifyMark()
         heapBlock->VerifyMark();
     });
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !this->IsAnyFinalizableBucket())
     {
         HeapBlockList::ForEach(this->sweepableHeapBlockList, [](TBlockType * heapBlock)

lib/Common/Memory/HeapBucket.h

@@ -225,7 +225,7 @@ protected:
     void DeleteHeapBlockList(TBlockType * list);
     static void DeleteEmptyHeapBlockList(TBlockType * list);
     static void DeleteHeapBlockList(TBlockType * list, Recycler * recycler);
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     static bool PushHeapBlockToSList(PSLIST_HEADER list, TBlockType * heapBlock);
     static TBlockType * PopHeapBlockFromSList(PSLIST_HEADER list);
     static ushort QueryDepthInterlockedSList(PSLIST_HEADER list);
@@ -274,10 +274,10 @@ protected:
     void EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size));
 
 
+    void AssertCheckHeapBlockNotInAnyList(TBlockType * heapBlock);
 #if DBG
     bool AllocatorsAreEmpty() const;
     bool HasPendingDisposeHeapBlocks() const;
-    void AssertCheckHeapBlockNotInAnyList(TBlockType * heapBlock);
 
     static void VerifyBlockConsistencyInList(TBlockType * heapBlock, RecyclerVerifyListConsistencyData& recyclerSweep);
     static void VerifyBlockConsistencyInList(TBlockType * heapBlock, RecyclerVerifyListConsistencyData const& recyclerSweep, SweepState state);
@@ -305,7 +305,7 @@ protected:
     TBlockType * heapBlockList;      // list of blocks that has free objects
 
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     PSLIST_HEADER allocableHeapBlockListHead;
     TBlockType * lastKnownNextAllocableBlockHead;
 #if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
@@ -373,7 +373,7 @@ HeapBucketT<TBlockType>::SweepBucket(RecyclerSweep& recyclerSweep, Fn sweepFn)
         // We should only queue up pending sweep if we are doing partial collect
         Assert(recyclerSweep.GetPendingSweepBlockList(this) == nullptr);
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         if (!this->AllocationsStartedDuringConcurrentSweep())
 #endif
 #endif

lib/Common/Memory/Recycler.cpp

@@ -3074,17 +3074,7 @@ Recycler::Sweep(bool concurrent)
 #if ENABLE_CONCURRENT_GC
     if (concurrent)
     {
-        bool needForceForground = false;
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && this->AllowAllocationsDuringConcurrentSweep())
-        {
-            needForceForground = !StartConcurrent(CollectionStateConcurrentSweepPass1);
-        }
-        else
-#endif
-        {
-            needForceForground = !StartConcurrent(CollectionStateConcurrentSweep);
-        }
+        bool needForceForground = !StartConcurrent(CollectionStateConcurrentSweep);
 
         if(needForceForground)
         {
@@ -3234,10 +3224,6 @@ Recycler::SweepHeap(bool concurrent, RecyclerSweep& recyclerSweep)
 #endif
 
         GCETW(GC_SETUPBACKGROUNDSWEEP_STOP, (this));
-
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        this->DoTwoPassConcurrentSweepPreCheck();
-#endif
     }
     else
     {
@@ -5736,22 +5722,11 @@ Recycler::FinishConcurrentCollect(CollectionFlags flags)
 #endif
 
 #if ENABLE_PARTIAL_GC
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && concurrent)
-        {
-            GCETW_INTERNAL(GC_STOP, (this, ETWEvent_ConcurrentRescan));
-        }
-#endif
         needConcurrentSweep = this->Sweep(rescanRootBytes, concurrent, true);
 #else
         needConcurrentSweep = this->Sweep(concurrent);
 #endif
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        if (!CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) || !concurrent)
-#endif
-        {
-            GCETW_INTERNAL(GC_STOP, (this, ETWEvent_ConcurrentRescan));
-        }
+        GCETW_INTERNAL(GC_STOP, (this, ETWEvent_ConcurrentRescan));
     }
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
     else if (collectionState == CollectionStateConcurrentSweepPass1Wait)
@@ -5994,9 +5969,19 @@ Recycler::DoBackgroundWork(bool forceForeground)
         Assert(this->enableConcurrentSweep);
 
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-        if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && this->AllowAllocationsDuringConcurrentSweep())
+        if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc) && !forceForeground)
         {
-            Assert(this->collectionState == CollectionStateConcurrentSweepPass1 || this->collectionState == CollectionStateConcurrentSweepPass2);
+            if (this->collectionState == CollectionStateConcurrentSweep)
+            {
+                this->DoTwoPassConcurrentSweepPreCheck();
+
+                if (this->AllowAllocationsDuringConcurrentSweep())
+                {
+                    this->collectionState = CollectionStateConcurrentSweepPass1;
+                }
+            }
+
+            Assert((!this->AllowAllocationsDuringConcurrentSweep() && this->collectionState == CollectionStateConcurrentSweep) || this->collectionState == CollectionStateConcurrentSweepPass1 || this->collectionState == CollectionStateConcurrentSweepPass2);
         }
         else
 #endif
@@ -6294,11 +6279,10 @@ Recycler::DoTwoPassConcurrentSweepPreCheck()
         // Do the actual 2-pass check only if the first 2 checks pass.
         if (this->allowAllocationsDuringConcurrentSweepForCollection)
         {
-            // TODO: akatti: Reenable this ETW event if needed.
             // We fire the ETW event only when the actual 2-pass check is performed. This is to avoid messing up ETL processing of test runs when in partial collect.
-            //GCETW_INTERNAL(GC_START, (this, ETWEvent_ConcurrentSweep_TwoPassSweepPreCheck));
+            GCETW_INTERNAL(GC_START, (this, ETWEvent_ConcurrentSweep_TwoPassSweepPreCheck));
             this->allowAllocationsDuringConcurrentSweepForCollection = this->autoHeap.DoTwoPassConcurrentSweepPreCheck();
-            //GCETW_INTERNAL(GC_STOP, (this, ETWEvent_ConcurrentSweep_TwoPassSweepPreCheck));
+            GCETW_INTERNAL(GC_STOP, (this, ETWEvent_ConcurrentSweep_TwoPassSweepPreCheck));
         }
     }
 }
@@ -6330,7 +6314,7 @@ Recycler::FinishSweepPrep()
 void
 Recycler::FinishConcurrentSweep()
 {
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     GCETW_INTERNAL(GC_START, (this, ETWEvent_ConcurrentSweep_FinishTwoPassSweep));
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
     {

lib/Common/Memory/SmallNormalHeapBucket.cpp

@@ -252,7 +252,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(RecyclerSweep& recycl
                 heapBlock->template SweepObjects<SweepMode_ConcurrentPartial>(recycler);
 
                 // page heap mode should never reach here, so don't check pageheap enabled or not
-                DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+                DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                 if (heapBlock->HasFreeObject())
                 {
                     AssertMsg(!HeapBlockList::Contains(heapBlock, partialSweptHeapBlockList), "The heap block already exists in the partialSweptHeapBlockList.");
@@ -280,7 +280,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(RecyclerSweep& recycl
             // We decided not to do a partial sweep.
             // Blocks in the pendingSweepList need to have a regular sweep.
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
             if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
             {
                 if (this->AllowAllocationsDuringConcurrentSweep() && !this->AllocationsStartedDuringConcurrentSweep())
@@ -293,7 +293,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(RecyclerSweep& recycl
 
             TBlockType * tail = SweepPendingObjects<SweepMode_Concurrent>(recycler, list);
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
             // During concurrent sweep if allocations were allowed, the heap blocks directly go into the SLIST of
             // allocable heap blocks. They will be returned to the heapBlockList at the end of the sweep.
             if (!this->AllowAllocationsDuringConcurrentSweep())
@@ -327,11 +327,11 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(Recycler * recycler,
         heapBlock->template SweepObjects<mode>(recycler);
         tail = heapBlock;
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         if (this->AllocationsStartedDuringConcurrentSweep())
         {
             Assert(!this->IsAnyFinalizableBucket());
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             // If we exhausted the free list during this sweep, we will need to send this block to the FullBlockList.
             if (heapBlock->HasFreeObject())
             {
@@ -353,7 +353,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(Recycler * recycler,
             }
             else
             {
-                DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+                DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                 heapBlock->SetNextBlock(this->fullBlockList);
                 this->fullBlockList = heapBlock;
 #ifdef RECYCLER_TRACE
@@ -392,13 +392,13 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
             callback(heapBlock, true);
 
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
             // During concurrent sweep if allocations were allowed, the heap blocks directly go into the SLIST of
             // allocable heap blocks. They will be returned to the heapBlockList at the end of the sweep.
             if(!allocationsAllowedDuringConcurrentSweep)
 #endif
             {
-                DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+                DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                 // Reuse the page
                 heapBlock->SetNextBlock(reuseBlocklist);
                 reuseBlocklist = heapBlock;
@@ -412,7 +412,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
             // Don't not reuse the page if it don't have much free memory.
             callback(heapBlock, false);
 
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             heapBlock->SetNextBlock(unusedBlockList);
             unusedBlockList = heapBlock;
 
@@ -430,7 +430,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
     Assert(this->GetRecycler()->inPartialCollectMode);
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (this->AllowAllocationsDuringConcurrentSweep() && !this->AllocationsStartedDuringConcurrentSweep())
     {
         Assert(!this->IsAnyFinalizableBucket());
@@ -444,12 +444,12 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
         this->partialHeapBlockList, this->AllocationsStartedDuringConcurrentSweep(),
         [this](TBlockType * heapBlock, bool isReused) 
     {
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
         if (isReused)
         {
             DebugOnly(heapBlock->blockNotReusedInPartialHeapBlockList = false);
 
-            DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+            DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
             if (heapBlock->HasFreeObject())
             {
                 if (this->AllocationsStartedDuringConcurrentSweep())
@@ -523,8 +523,8 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
                 recycler->PrintBlockStatus(this, heapBlock, _u("[**20**] calling SweepObjects."));
 #endif
                 heapBlock->template SweepObjects<SweepMode_InThread>(recycler);
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
-                DebugOnly(AssertCheckHeapBlockNotInAnyList(heapBlock));
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
+                DebugOnly(this->AssertCheckHeapBlockNotInAnyList(heapBlock));
                 if (heapBlock->HasFreeObject())
                 {
                     if (this->AllocationsStartedDuringConcurrentSweep())
@@ -576,7 +576,7 @@ SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& rec
 
     RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
 
-#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST
     if (!this->AllocationsStartedDuringConcurrentSweep())
 #endif
     {
@@ -714,7 +714,7 @@ SmallNormalHeapBucketBase<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount
 
 #if ENABLE_CONCURRENT_GC
 #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP
-#if SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
+#if SUPPORT_WIN32_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))
     {
         allocatingDuringConcurrentSweep = true;
@@ -722,7 +722,7 @@ SmallNormalHeapBucketBase<TBlockType>::GetNonEmptyHeapBlockCount(bool checkCount
 #endif
 #endif
 #endif
-    RECYCLER_SLOW_CHECK(Assert(!checkCount || this->heapBlockCount == currentHeapBlockCount /*|| (this->heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)*/));
+    RECYCLER_SLOW_CHECK(Assert(!checkCount || this->heapBlockCount == currentHeapBlockCount || (this->heapBlockCount >= 65535 && allocatingDuringConcurrentSweep)));
     return currentHeapBlockCount;
 }
 #endif