update dump fragmentation stats

Brings -DumpFragmentationStats results up-to-date with JD.

- Corrected heapBlock->isInAllocator handling during stats aggregation.
  Only aggregate isInAllocator blocks when processing allocatorHead list.
  The same blocks could appear in other lists but won't be counted again.

- To help combine stats from heapInfo new blocks and in bucket blocks,
  added `memStats` field to each bucket. Accumulate heapInfo new blocks
  stats in a special pre-aggregate pass. Keep the data and combine with
  in bucket blocks.

- Added LargeBlock stats aggregation.

lib/Common/Memory/HeapBlock.cpp

@@ -7,6 +7,95 @@
 #include <cxxabi.h>
 #endif
 
+
+#if ENABLE_MEM_STATS
+MemStats::MemStats()
+    : objectByteCount(0), totalByteCount(0)
+{}
+
+void MemStats::Reset()
+{
+    objectByteCount = 0;
+    totalByteCount = 0;
+}
+
+size_t MemStats::FreeBytes() const
+{
+    return totalByteCount - objectByteCount;
+}
+
+double MemStats::UsedRatio() const
+{
+    return (double)objectByteCount / totalByteCount;
+}
+
+void MemStats::Aggregate(const MemStats& other)
+{
+    objectByteCount += other.objectByteCount;
+    totalByteCount += other.totalByteCount;
+}
+
+#ifdef DUMP_FRAGMENTATION_STATS
+HeapBucketStats::HeapBucketStats()
+    : totalBlockCount(0), objectCount(0), finalizeCount(0)
+{}
+
+void HeapBucketStats::Reset()
+{
+    MemStats::Reset();
+    totalBlockCount = 0;
+    objectCount = 0;
+    finalizeCount = 0;
+}
+
+void HeapBucketStats::Dump() const
+{
+    Output::Print(_u("%5d %7d %7d %11lu %11lu %11lu   %6.2f%%\n"),
+        totalBlockCount, objectCount, finalizeCount,
+        static_cast<ULONG>(objectByteCount),
+        static_cast<ULONG>(FreeBytes()),
+        static_cast<ULONG>(totalByteCount),
+        UsedRatio() * 100);
+}
+#endif
+
+void HeapBucketStats::PreAggregate()
+{
+    // When first enter Pre-Aggregate state, clear data and mark state.
+    if (!(totalByteCount & 1))
+    {
+        Reset();
+        totalByteCount |= 1;
+    }
+}
+
+void HeapBucketStats::BeginAggregate()
+{
+    // If was Pre-Aggregate state, keep data and clear state
+    if (totalByteCount & 1)
+    {
+        totalByteCount &= ~1;
+    }
+    else
+    {
+        Reset();
+    }
+}
+
+void HeapBucketStats::Aggregate(const HeapBucketStats& other)
+{
+    MemStats::Aggregate(other);
+#ifdef DUMP_FRAGMENTATION_STATS
+    totalBlockCount += other.totalBlockCount;
+    objectCount += other.objectCount;
+    finalizeCount += other.finalizeCount;
+#endif
+}
+#endif  // ENABLE_MEM_STATS
+
+//========================================================================================================
+// HeapBlock
+//========================================================================================================
 template <typename TBlockAttributes>
 SmallNormalHeapBlockT<TBlockAttributes> *
 HeapBlock::AsNormalBlock()
@@ -887,7 +976,7 @@ void HeapBlock::PrintVerifyMarkFailure(Recycler* recycler, char* objectAddress,
             }
             else
             {
-                auto dumpFalsePositive = [&]() 
+                auto dumpFalsePositive = [&]()
                 {
                     if (CONFIG_FLAG(Verbose))
                     {
@@ -904,7 +993,7 @@ void HeapBlock::PrintVerifyMarkFailure(Recycler* recycler, char* objectAddress,
                 //TODO: (leish)(swb) analyze pdb to check if the field is a pointer field or not
                 Output::Print(_u("Missing Barrier\nOn type %S+0x%x\n"), typeName, offset);
             }
-        }        
+        }
 
 
         targetStartAddress = target - targetOffset;
@@ -1959,19 +2048,17 @@ template <class TBlockAttributes>
 void
 SmallHeapBlockT<TBlockAttributes>::AggregateBlockStats(HeapBucketStats& stats, bool isAllocatorBlock, FreeObject* freeObjectList, bool isBumpAllocated)
 {
+    if (this->segment == nullptr || this->IsInAllocator() != isAllocatorBlock)
+    {
+        return;  // skip empty blocks, or blocks mismatching isInAllocator to avoid double count
+    }
+
     DUMP_FRAGMENTATION_STATS_ONLY(stats.totalBlockCount++);
 
     ushort blockObjectCount = this->objectCount;
     BVIndex blockFreeCount = this->GetFreeBitVector()->Count();
     ushort blockObjectSize = this->objectSize;
 
-    if (this->segment == nullptr)
-    {
-        DUMP_FRAGMENTATION_STATS_ONLY(stats.emptyBlockCount++);
-        blockObjectCount = 0;
-        blockFreeCount = 0;
-    }
-
     uint objectCount = 0;
     if (isBumpAllocated)
     {
@@ -1999,23 +2086,9 @@ SmallHeapBlockT<TBlockAttributes>::AggregateBlockStats(HeapBucketStats& stats, b
         }
     }
 
-    // If we have a block that's on the allocator, it could also be on the heap block list
-    // In that case, we need to make sure we don't double-count this. To do that, we take out
-    // the block's allocatorCount/freeCount and adjust it later when we see the block
-    if (isAllocatorBlock)
-    {
-        objectCount -= blockObjectCount;
-        objectCount += blockFreeCount;
-    }
-
-    // Don't count empty blocks as allocable
-    if (this->segment != nullptr)
-    {
-        stats.totalByteCount += this->GetPageCount() * AutoSystemInfo::PageSize;
-    }
-
     DUMP_FRAGMENTATION_STATS_ONLY(stats.objectCount += objectCount);
     stats.objectByteCount += (objectCount * blockObjectSize);
+    stats.totalByteCount += this->GetPageCount() * AutoSystemInfo::PageSize;
 
 #ifdef DUMP_FRAGMENTATION_STATS
     if (!isAllocatorBlock)
@@ -2023,7 +2096,6 @@ SmallHeapBlockT<TBlockAttributes>::AggregateBlockStats(HeapBucketStats& stats, b
         if (this->IsAnyFinalizableBlock())
         {
             auto finalizableBlock = this->AsFinalizableBlock<TBlockAttributes>();
-            stats.finalizeBlockCount++;
             stats.finalizeCount += (finalizableBlock->GetFinalizeCount());
         }
     }

lib/Common/Memory/HeapBlock.h

@@ -43,24 +43,29 @@ struct MemStats
     size_t objectByteCount;
     size_t totalByteCount;
 
-    MemStats() : objectByteCount(0), totalByteCount(0) {}
+    MemStats();
 
-    void Aggregate(const MemStats& other)
-    {
-        objectByteCount += other.objectByteCount;
-        totalByteCount += other.totalByteCount;
-    }
+    void Reset();
+    size_t FreeBytes() const;
+    double UsedRatio() const;
+    void Aggregate(const MemStats& other);
 };
 
 struct HeapBucketStats: MemStats
 {
 #ifdef DUMP_FRAGMENTATION_STATS
     uint totalBlockCount;
-    uint emptyBlockCount;
-    uint finalizeBlockCount;
     uint objectCount;
     uint finalizeCount;
+
+    HeapBucketStats();
+    void Reset();
+    void Dump() const;
 #endif
+
+    void PreAggregate();
+    void BeginAggregate();
+    void Aggregate(const HeapBucketStats& other);
 };
 
 #ifdef DUMP_FRAGMENTATION_STATS
@@ -68,7 +73,7 @@ struct HeapBucketStats: MemStats
 #else
 #define DUMP_FRAGMENTATION_STATS_ONLY(x)
 #endif
-#endif
+#endif  // ENABLE_MEM_STATS
 
 #if defined(PROFILE_RECYCLER_ALLOC) || defined(RECYCLER_MEMORY_VERIFY) || defined(MEMSPECT_TRACKING) || defined(RECYCLER_PERF_COUNTERS) || defined(ETW_MEMORY_TRACKING)
 #define RECYCLER_TRACK_NATIVE_ALLOCATED_OBJECTS

lib/Common/Memory/HeapBucket.cpp

@@ -1431,8 +1431,10 @@ HeapBucketT<TBlockType>::Check(bool checkCount)
 #if ENABLE_MEM_STATS
 template <typename TBlockType>
 void
-HeapBucketT<TBlockType>::AggregateBucketStats(HeapBucketStats& stats)
+HeapBucketT<TBlockType>::AggregateBucketStats()
 {
+    HeapBucket::AggregateBucketStats();  // call super
+
     auto allocatorHead = &this->allocatorHead;
     auto allocatorCurr = allocatorHead;
 
@@ -1441,16 +1443,15 @@ HeapBucketT<TBlockType>::AggregateBucketStats(HeapBucketStats& stats)
         TBlockType* allocatorHeapBlock = allocatorCurr->GetHeapBlock();
         if (allocatorHeapBlock)
         {
-            allocatorHeapBlock->AggregateBlockStats(stats, true, allocatorCurr->freeObjectList, allocatorCurr->endAddress != 0);
+            allocatorHeapBlock->AggregateBlockStats(this->memStats, true, allocatorCurr->freeObjectList, allocatorCurr->endAddress != 0);
         }
         allocatorCurr = allocatorCurr->GetNext();
     } while (allocatorCurr != allocatorHead);
 
-    auto blockStatsAggregator = [&stats](TBlockType* heapBlock) {
-        heapBlock->AggregateBlockStats(stats);
+    auto blockStatsAggregator = [this](TBlockType* heapBlock) {
+        heapBlock->AggregateBlockStats(this->memStats);
     };
 
-    HeapBlockList::ForEach(emptyBlockList, blockStatsAggregator);
     HeapBlockList::ForEach(fullBlockList, blockStatsAggregator);
 #if ENABLE_CONCURRENT_GC && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP && SUPPORT_WIN32_SLIST && ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP_USE_SLIST
     if (CONFIG_FLAG_RELEASE(EnableConcurrentSweepAlloc))

lib/Common/Memory/HeapBucket.h

@@ -83,6 +83,7 @@ protected:
 #endif
 
 #ifdef RECYCLER_PAGE_HEAP
+protected:
     bool isPageHeapEnabled;
 public:
     inline bool IsPageHeapEnabled(ObjectInfoBits attributes) const
@@ -91,6 +92,27 @@ public:
         return isPageHeapEnabled && ((attributes & ClientTrackableObjectBits) == 0);
     }
 #endif
+
+#if ENABLE_MEM_STATS
+protected:
+    HeapBucketStats memStats;  // mem stats per bucket
+public:
+    const HeapBucketStats& GetMemStats() const { return memStats; }
+
+    template <typename TBlockType>
+    void PreAggregateBucketStats(TBlockType* heapBlock)
+    {
+        Assert(heapBlock->heapBucket == this);
+        memStats.PreAggregate();
+        heapBlock->AggregateBlockStats(memStats);
+    }
+
+    void AggregateBucketStats()
+    {
+        memStats.BeginAggregate();  // Begin aggregate, clear if needed
+    }
+#endif
+
     Recycler * GetRecycler() const;
 
     template <typename TBlockType>
@@ -143,8 +165,9 @@ public:
     void ScanNewImplicitRoots(Recycler * recycler);
 
 #if ENABLE_MEM_STATS
-    void AggregateBucketStats(HeapBucketStats& stats);
+    void AggregateBucketStats();
 #endif
+
     uint Rescan(Recycler * recycler, RescanFlags flags);
 #if ENABLE_CONCURRENT_GC
     void MergeNewHeapBlock(TBlockType * heapBlock);

lib/Common/Memory/HeapInfo.cpp

@@ -1149,6 +1149,9 @@ HeapInfo::Rescan(RescanFlags flags)
     return scannedPageCount;
 }
 
+
+#if ENABLE_MEM_STATS
+
 #ifdef DUMP_FRAGMENTATION_STATS
 template <ObjectInfoBits TBucketType>
 struct DumpBucketTypeName { static char16 name[]; };
@@ -1164,13 +1167,26 @@ template<> char16 DumpBlockTypeName<SmallAllocationBlockAttributes>::name[] = _u
 template<> char16 DumpBlockTypeName<MediumAllocationBlockAttributes>::name[] = _u("(M)");
 #endif
 
-#if ENABLE_MEM_STATS
+template <class TBlockType>
+void PreAggregateBucketStats(TBlockType* list)
+{
+    HeapBlockList::ForEach(list, [](auto heapBlock)
+    {
+        // Process blocks not in allocator in pre-pass. They are not put into buckets yet.
+        if (!heapBlock->IsInAllocator())
+        {
+            heapBlock->heapBucket->PreAggregateBucketStats(heapBlock);
+        }
+    });
+}
+
 template <class TBlockAttributes, ObjectInfoBits TBucketType>
-void GetBucketStats(HeapBucketGroup<TBlockAttributes>& group, MemStats& total, bool dumpFragmentationStats)
+void GetBucketStats(HeapBucketGroup<TBlockAttributes>& group, HeapBucketStats& total, bool dumpFragmentationStats)
 {
     auto& bucket = group.GetBucket<TBucketType>();
-    HeapBucketStats stats = {};
-    bucket.AggregateBucketStats(stats);
+    bucket.AggregateBucketStats();
+
+    const auto& stats = bucket.GetMemStats();
     total.Aggregate(stats);
 
 #ifdef DUMP_FRAGMENTATION_STATS
@@ -1178,10 +1194,23 @@ void GetBucketStats(HeapBucketGroup<TBlockAttributes>& group, MemStats& total, b
     {
         Output::Print(_u("%-7s%s %4d : "),
             DumpBucketTypeName<TBucketType>::name, DumpBlockTypeName<TBlockAttributes>::name, bucket.GetSizeCat());
-        Output::Print(_u("%7d %7d %7d %7d %7d %10lu %10lu\n"),
-            stats.totalBlockCount, stats.finalizeBlockCount, stats.emptyBlockCount,
-            stats.objectCount, stats.finalizeCount,
-            static_cast<ulong>(stats.objectByteCount), static_cast<ulong>(stats.totalByteCount));
+        stats.Dump();
+    }
+#endif
+}
+
+void GetBucketStats(LargeHeapBucket& bucket, HeapBucketStats& total, bool dumpFragmentationStats)
+{
+    bucket.AggregateBucketStats();
+
+    const auto& stats = bucket.GetMemStats();
+    total.Aggregate(stats);
+
+#ifdef DUMP_FRAGMENTATION_STATS
+    if (dumpFragmentationStats && stats.totalByteCount > 0)
+    {
+        Output::Print(_u("Large           : "));
+        stats.Dump();
     }
 #endif
 }
@@ -1197,7 +1226,7 @@ HeapInfo::ReportMemStats()
         dumpFragmentationStats = true;
         Output::Print(_u("[FRAG %d] Post-Collection State\n"), ::GetTickCount());
         Output::Print(_u("---------------------------------------------------------------------------------------\n"));
-        Output::Print(_u("                     #Blk  #FinBlk  #EmpBlk  #Objs    #Fin    ObjBytes  TotalBytes\n"));
+        Output::Print(_u("                  #Blk   #Objs    #Fin     ObjBytes   FreeBytes  TotalBytes UsedPercent\n"));
         Output::Print(_u("---------------------------------------------------------------------------------------\n"));
     }
 #endif
@@ -1207,7 +1236,34 @@ HeapInfo::ReportMemStats()
         return;
     }
 
-    MemStats total;
+#if ENABLE_CONCURRENT_GC
+    // Pre aggregate pass on all the heap blocks that are not merged into bucket's lists yet
+    PreAggregateBucketStats(this->newNormalHeapBlockList);
+    PreAggregateBucketStats(this->newLeafHeapBlockList);
+    PreAggregateBucketStats(this->newFinalizableHeapBlockList);
+#ifdef RECYCLER_WRITE_BARRIER
+    PreAggregateBucketStats(this->newNormalWithBarrierHeapBlockList);
+    PreAggregateBucketStats(this->newFinalizableWithBarrierHeapBlockList);
+#endif
+#ifdef RECYCLER_VISITED_HOST
+    PreAggregateBucketStats(this->newRecyclerVisitedHostHeapBlockList);
+#endif
+
+#if defined(BUCKETIZE_MEDIUM_ALLOCATIONS) && SMALLBLOCK_MEDIUM_ALLOC
+    PreAggregateBucketStats(this->newMediumNormalHeapBlockList);
+    PreAggregateBucketStats(this->newMediumLeafHeapBlockList);
+    PreAggregateBucketStats(this->newMediumFinalizableHeapBlockList);
+#ifdef RECYCLER_WRITE_BARRIER
+    PreAggregateBucketStats(this->newMediumNormalWithBarrierHeapBlockList);
+    PreAggregateBucketStats(this->newMediumFinalizableWithBarrierHeapBlockList);
+#endif
+#ifdef RECYCLER_VISITED_HOST
+    PreAggregateBucketStats(this->newMediumRecyclerVisitedHostHeapBlockList);
+#endif
+#endif
+#endif  // ENABLE_CONCURRENT_GC
+
+    HeapBucketStats total;
     for (uint i = 0; i < HeapConstants::BucketCount; i++)
     {
         GetBucketStats<SmallAllocationBlockAttributes, NoBit>(heapBuckets[i], total, dumpFragmentationStats);
@@ -1238,9 +1294,20 @@ HeapInfo::ReportMemStats()
     }
 #endif
 
-    // TODO: Large bucket
-}
+    GetBucketStats(largeObjectBucket, total, dumpFragmentationStats);
+
+#ifdef DUMP_FRAGMENTATION_STATS
+    if (dumpFragmentationStats)
+    {
+        Output::Print(_u("---------------------------------------------------------------------------------------\n"));
+        Output::Print(_u("Total           : "));
+        total.Dump();
+    }
 #endif
+}
+
+#endif  // ENABLE_MEM_STATS
+
 
 #if ENABLE_PARTIAL_GC
 void

lib/Common/Memory/LargeHeapBlock.cpp

@@ -1950,6 +1950,30 @@ LargeHeapBlock::EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunctio
     }
 }
 
+#if ENABLE_MEM_STATS
+void
+LargeHeapBlock::AggregateBlockStats(HeapBucketStats& stats)
+{
+    DUMP_FRAGMENTATION_STATS_ONLY(uint objectCount = 0);
+    size_t objectSize = 0;
+    for (uint i = 0; i < allocCount; i++)
+    {
+        LargeObjectHeader * header = this->GetHeaderByIndex(i);
+        if (header)
+        {
+            DUMP_FRAGMENTATION_STATS_ONLY(objectCount++);
+            objectSize += header->objectSize;
+        }
+    }
+
+    DUMP_FRAGMENTATION_STATS_ONLY(stats.totalBlockCount++);
+    DUMP_FRAGMENTATION_STATS_ONLY(stats.objectCount += objectCount);
+    DUMP_FRAGMENTATION_STATS_ONLY(stats.finalizeCount += this->finalizeCount);
+
+    stats.objectByteCount += objectSize;
+    stats.totalByteCount += AutoSystemInfo::PageSize * pageCount;
+}
+#endif  // ENABLE_MEM_STATS
 
 uint
 LargeHeapBlock::GetMaxLargeObjectCount(size_t pageCount, size_t firstAllocationSize)

lib/Common/Memory/LargeHeapBlock.h

@@ -186,6 +186,10 @@ public:
 
     void EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size));
 
+#if ENABLE_MEM_STATS
+    void AggregateBlockStats(HeapBucketStats& stats);
+#endif
+
 #ifdef RECYCLER_SLOW_CHECK_ENABLED
     void Check(bool expectFull, bool expectPending);
 #endif

lib/Common/Memory/LargeHeapBucket.cpp

@@ -1123,3 +1123,24 @@ LargeHeapBucket::VerifyMark()
 }
 #endif
 
+#if ENABLE_MEM_STATS
+void
+LargeHeapBucket::AggregateBucketStats()
+{
+    HeapBucket::AggregateBucketStats();  // call super
+
+    auto blockStatsAggregator = [this](LargeHeapBlock* largeHeapBlock) {
+        largeHeapBlock->AggregateBlockStats(this->memStats);
+    };
+
+    HeapBlockList::ForEach(largeBlockList, blockStatsAggregator);
+    HeapBlockList::ForEach(fullLargeBlockList, blockStatsAggregator);
+    HeapBlockList::ForEach(pendingDisposeLargeBlockList, blockStatsAggregator);
+#if ENABLE_CONCURRENT_GC
+    HeapBlockList::ForEach(pendingSweepLargeBlockList, blockStatsAggregator);
+#if ENABLE_PARTIAL_GC
+    HeapBlockList::ForEach(partialSweptLargeBlockList, blockStatsAggregator);
+#endif
+#endif
+}
+#endif

lib/Common/Memory/LargeHeapBucket.h

@@ -94,6 +94,10 @@ public:
 #endif
 #endif
 
+#if ENABLE_MEM_STATS
+    void AggregateBucketStats();
+#endif
+
 private:
     char * SnailAlloc(Recycler * recycler, DECLSPEC_GUARD_OVERFLOW size_t sizeCat, size_t size, ObjectInfoBits attributes, bool nothrow);
     char * TryAlloc(Recycler * recycler, DECLSPEC_GUARD_OVERFLOW size_t sizeCat, ObjectInfoBits attributes);

lib/Common/Memory/SmallFinalizableHeapBucket.cpp

@@ -83,12 +83,12 @@ SmallFinalizableHeapBucketBaseT<TBlockType>::ResetMarks(ResetMarkFlags flags)
 #if ENABLE_MEM_STATS
 template <class TBlockType>
 void
-SmallFinalizableHeapBucketBaseT<TBlockType>::AggregateBucketStats(HeapBucketStats& stats)
+SmallFinalizableHeapBucketBaseT<TBlockType>::AggregateBucketStats()
 {
-    __super::AggregateBucketStats(stats);
+    __super::AggregateBucketStats();
 
-    HeapBlockList::ForEach(pendingDisposeList, [&stats](TBlockType* heapBlock) {
-        heapBlock->AggregateBlockStats(stats);
+    HeapBlockList::ForEach(pendingDisposeList, [this](TBlockType* heapBlock) {
+        heapBlock->AggregateBlockStats(this->memStats);
     });
 }
 #endif

lib/Common/Memory/SmallFinalizableHeapBucket.h

@@ -20,7 +20,7 @@ public:
     static void FinalizeHeapBlockList(THeapBlockType * list);
 
 #if ENABLE_MEM_STATS
-    void AggregateBucketStats(HeapBucketStats& stats);
+    void AggregateBucketStats();
 #endif
 protected:
     void EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size));

lib/Common/Memory/SmallNormalHeapBucket.cpp

@@ -19,15 +19,15 @@ SmallNormalHeapBucketBase<TBlockType>::SmallNormalHeapBucketBase()
 #if ENABLE_MEM_STATS
 template <typename TBlockType>
 void
-SmallNormalHeapBucketBase<TBlockType>::AggregateBucketStats(HeapBucketStats& stats)
+SmallNormalHeapBucketBase<TBlockType>::AggregateBucketStats()
 {
-    __super::AggregateBucketStats(stats);
+    __super::AggregateBucketStats();
 
-    HeapBlockList::ForEach(partialHeapBlockList, [&stats](TBlockType* heapBlock) {
-        heapBlock->AggregateBlockStats(stats);
+    HeapBlockList::ForEach(partialHeapBlockList, [this](TBlockType* heapBlock) {
+        heapBlock->AggregateBlockStats(this->memStats);
     });
-    HeapBlockList::ForEach(partialSweptHeapBlockList, [&stats](TBlockType* heapBlock) {
-        heapBlock->AggregateBlockStats(stats);
+    HeapBlockList::ForEach(partialSweptHeapBlockList, [this](TBlockType* heapBlock) {
+        heapBlock->AggregateBlockStats(this->memStats);
     });
 }
 #endif

lib/Common/Memory/SmallNormalHeapBucket.h

@@ -21,7 +21,7 @@ public:
 #endif
 
 #if ENABLE_MEM_STATS
-    void AggregateBucketStats(HeapBucketStats& stats);
+    void AggregateBucketStats();
 #endif
 protected:
     template <class TBlockAttributes>