//------------------------------------------------------------------------------------------------------- // Copyright (C) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. //------------------------------------------------------------------------------------------------------- #pragma once #include "CollectionState.h" #include "RecyclerTelemetryInfo.h" #include "RecyclerWaitReason.h" #include "Common/ObservableValue.h" #include "CollectionFlags.h" namespace Js { class Profiler; enum Phase: unsigned short; }; namespace JsUtil { class ThreadService; }; #ifdef STACK_BACK_TRACE class StackBackTraceNode; #endif class ScriptEngineBase; class JavascriptThreadService; #ifdef PROFILE_MEM struct RecyclerMemoryData; #endif class ThreadContext; namespace Memory { template class RecyclerRootPtr; class AutoBooleanToggle { public: AutoBooleanToggle(bool * b, bool value = true, bool valueMayChange = false) : b(b) { Assert(!(*b)); *b = value; #if DBG this->value = value; this->valueMayChange = valueMayChange; #endif } ~AutoBooleanToggle() { if (b) { Assert(valueMayChange || *b == value); *b = false; } } void Leave() { Assert(valueMayChange || *b == value); *b = false; b = nullptr; } private: bool * b; #if DBG bool value; bool valueMayChange; #endif }; template class AutoRestoreValue { public: AutoRestoreValue(T* var, const T& val): variable(var) { Assert(var); oldValue = (*variable); (*variable) = val; #ifdef DEBUG debugSetValue = val; #endif } ~AutoRestoreValue() { Assert((*variable) == debugSetValue); (*variable) = oldValue; } private: #ifdef DEBUG T debugSetValue; #endif T* variable; T oldValue; }; class Recycler; class RecyclerScanMemoryCallback { public: RecyclerScanMemoryCallback(Recycler* recycler) : recycler(recycler) {} void operator()(void** obj, size_t byteCount); private: Recycler* recycler; }; template struct InfoBitsWrapper{}; #if ENABLE_WEAK_REFERENCE_REGIONS template static constexpr bool is_pointer = false; template static constexpr bool is_pointer = true; template class RecyclerWeakReferenceRegionItem { static_assert(is_pointer, "Weak references must be to pointer types"); friend class Recycler; public: RecyclerWeakReferenceRegionItem() : ptr(T()), heapBlock(nullptr) {}; operator T() const { return ptr; }; T operator=(T newPtr) { Assert(ptr == nullptr); // For safety with concurrent marking, only allow setting the pointer to non-null from null heapBlock = nullptr; return ptr = newPtr; }; void Clear() { heapBlock = nullptr; ptr = nullptr; }; private: RecyclerWeakReferenceRegionItem(RecyclerWeakReferenceRegionItem&) = delete; FieldNoBarrier(T) ptr; FieldNoBarrier(HeapBlock*) heapBlock; // Note: the low bit of the heapBlock is used for background marking }; class RecyclerWeakReferenceRegion { friend class Recycler; public: RecyclerWeakReferenceRegionItem* GetPtr() const { return ptr; } size_t GetCount() const { return count; } HeapBlock* GetHeapBlock() const { return arrayHeapBlock; } private: FieldNoBarrier(RecyclerWeakReferenceRegionItem*) ptr; FieldNoBarrier(size_t) count; FieldNoBarrier(HeapBlock*) arrayHeapBlock; }; #endif // Allocation macro #define RecyclerNew(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocInlined, T, __VA_ARGS__) #define RecyclerNewPlus(recycler,size,T,...) AllocatorNewPlus(Recycler, recycler, size, T, __VA_ARGS__) #define RecyclerNewPlusZ(recycler,size,T,...) AllocatorNewPlusZ(Recycler, recycler, size, T, __VA_ARGS__) #define RecyclerNewZ(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocZeroInlined, T, __VA_ARGS__) #define RecyclerNewStruct(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocInlined, T) #define RecyclerNewStructZ(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocZeroInlined, T) #define RecyclerNewStructPlus(recycler,size,T) AllocatorNewStructPlus(Recycler, recycler, size, T) #define RecyclerNewArray(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, Alloc, T, count) #define RecyclerNewArrayZ(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocZero, T, count) #define RecyclerNewFinalized(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedInlined, T, __VA_ARGS__))) #define RecyclerNewFinalizedPlus(recycler, size, T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocFinalized, size, T, __VA_ARGS__))) #define RecyclerNewTracked(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedInlined, T, __VA_ARGS__))) #define RecyclerNewEnumClass(recycler, enumClass, T, ...) new (TRACK_ALLOC_INFO(static_cast(recycler), T, Recycler, 0, (size_t)-1), InfoBitsWrapper()) T(__VA_ARGS__) #define RecyclerNewWithInfoBits(recycler, infoBits, T, ...) new (TRACK_ALLOC_INFO(static_cast(recycler), T, Recycler, 0, (size_t)-1), InfoBitsWrapper()) T(__VA_ARGS__) #define RecyclerNewFinalizedClientTracked(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedClientTrackedInlined, T, __VA_ARGS__))) #if defined(RECYCLER_WRITE_BARRIER_ALLOC) #define RecyclerNewWithBarrier(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocWithBarrier, T, __VA_ARGS__) #define RecyclerNewWithBarrierPlus(recycler,size,T,...) AllocatorNewPlusBase(Recycler, recycler, AllocWithBarrier, size, T, __VA_ARGS__) #define RecyclerNewWithBarrierPlusZ(recycler,size,T,...) AllocatorNewPlusBase(Recycler, recycler, AllocZeroWithBarrier, size, T, __VA_ARGS__) #define RecyclerNewWithBarrierZ(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocZeroWithBarrier, T, __VA_ARGS__) #define RecyclerNewWithBarrierStruct(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocWithBarrier, T) #define RecyclerNewWithBarrierStructZ(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocZeroWithBarrier, T) #define RecyclerNewWithBarrierStructPlus(recycler,size,T) AllocatorNewStructPlusBase(Recycler, recycler, AllocWithBarrier, size, T) #define RecyclerNewWithBarrierArray(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocWithBarrier, T, count) #define RecyclerNewWithBarrierArrayZ(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocZeroWithBarrier, T, count) #define RecyclerNewWithBarrierFinalized(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedWithBarrierInlined, T, __VA_ARGS__))) #define RecyclerNewWithBarrierFinalizedPlus(recycler, size, T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocFinalizedWithBarrier, size, T, __VA_ARGS__))) #define RecyclerNewWithBarrierTracked(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedWithBarrierInlined, T, __VA_ARGS__))) #define RecyclerNewWithBarrierEnumClass(recycler, enumClass, T, ...) new (TRACK_ALLOC_INFO(static_cast(recycler), T, Recycler, 0, (size_t)-1), InfoBitsWrapper<(ObjectInfoBits)(enumClass | WithBarrierBit)>()) T(__VA_ARGS__) #define RecyclerNewWithBarrierWithInfoBits(recycler, infoBits, T, ...) new (TRACK_ALLOC_INFO(static_cast(recycler), T, Recycler, 0, (size_t)-1), InfoBitsWrapper<(ObjectInfoBits)(infoBits | WithBarrierBit)>()) T(__VA_ARGS__) #define RecyclerNewWithBarrierFinalizedClientTracked(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedClientTrackedWithBarrierInlined, T, __VA_ARGS__))) #endif #ifndef RECYCLER_WRITE_BARRIER #define RecyclerNewWithBarrier RecyclerNew #define RecyclerNewWithBarrierPlus RecyclerNewPlus #define RecyclerNewWithBarrierPlusZ RecyclerNewPlusZ #define RecyclerNewWithBarrierZ RecyclerNewZ #define RecyclerNewWithBarrierStruct RecyclerNewStruct #define RecyclerNewWithBarrierStructZ RecyclerNewStructZ #define RecyclerNewWithBarrierStructPlus RecyclerNewStructPlus #define RecyclerNewWithBarrierArray RecyclerNewArray #define RecyclerNewWithBarrierArrayZ RecyclerNewArrayZ #define RecyclerNewWithBarrierFinalized RecyclerNewFinalized #define RecyclerNewWithBarrierFinalizedPlus RecyclerNewFinalizedPlus #define RecyclerNewWithBarrierTracked RecyclerNewTracked #define RecyclerNewWithBarrierEnumClass RecyclerNewEnumClass #define RecyclerNewWithBarrierWithInfoBits RecyclerNewWithInfoBits #define RecyclerNewWithBarrierFinalizedClientTracked RecyclerNewFinalizedClientTracked #endif // Leaf allocators #define RecyclerNewLeaf(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocLeafInlined, T, __VA_ARGS__) #define RecyclerNewLeafZ(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocLeafZeroInlined, T, __VA_ARGS__) #define RecyclerNewPlusLeaf(recycler,size,T,...) AllocatorNewPlusLeaf(Recycler, recycler, size, T, __VA_ARGS__) #define RecyclerNewPlusLeafZ(recycler,size,T,...) AllocatorNewPlusLeafZ(Recycler, recycler, size, T, __VA_ARGS__) #define RecyclerNewStructLeaf(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocLeafInlined, T) #define RecyclerNewStructLeafZ(recycler,T) AllocatorNewStructBase(Recycler, recycler, AllocLeafZeroInlined, T) #define RecyclerNewArrayLeafZ(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocLeafZero, T, count) #define RecyclerNewArrayLeaf(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocLeaf, T, count) #define RecyclerNewFinalizedLeaf(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedLeafInlined, T, __VA_ARGS__))) #define RecyclerNewFinalizedLeafPlus(recycler, size, T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocFinalizedLeaf, size, T, __VA_ARGS__))) #define RecyclerNewTrackedLeaf(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedLeafInlined, T, __VA_ARGS__))) #define RecyclerNewTrackedLeafPlusZ(recycler,size,T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocZeroTrackedLeafInlined, size, T, __VA_ARGS__))) #ifdef RECYCLER_VISITED_HOST // We need to track these allocations. The RecyclerVisitedHost* object allocation APIs don't provide us with the type of the objects being allocated. Use the DummyVTableObject type used elsewhere to track the allocations. #define RecyclerAllocVisitedHostTracedAndFinalized(recycler,size) (TRACK_ALLOC_INFO(recycler, DummyVTableObject, Recycler, size, (size_t)-1))->AllocVisitedHost(size) #define RecyclerAllocVisitedHostFinalized(recycler,size) (TRACK_ALLOC_INFO(recycler, DummyVTableObject, Recycler, size, (size_t)-1))->AllocVisitedHost(size) #define RecyclerAllocVisitedHostTraced(recycler,size) (TRACK_ALLOC_INFO(recycler, DummyVTableObject, Recycler, size, (size_t)-1))->AllocVisitedHost(size) #define RecyclerAllocLeaf(recycler,size) (TRACK_ALLOC_INFO(recycler, DummyVTableObject, Recycler, size, (size_t)-1))->AllocVisitedHost(size) #endif #ifdef TRACE_OBJECT_LIFETIME #define RecyclerNewLeafTrace(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocLeafTrace, T, __VA_ARGS__) #define RecyclerNewLeafZTrace(recycler,T,...) AllocatorNewBase(Recycler, recycler, AllocLeafZeroTrace, T, __VA_ARGS__) #define RecyclerNewPlusLeafTrace(recycler,size,T,...) AllocatorNewPlusBase(Recycler, recycler, AllocLeafTrace, size, T, __VA_ARGS__) #define RecyclerNewArrayLeafZTrace(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocLeafZeroTrace, T, count) #define RecyclerNewArrayTrace(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocTrace, T, count) #define RecyclerNewArrayZTrace(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocZeroTrace, T, count) #define RecyclerNewArrayLeafTrace(recycler,T,count) AllocatorNewArrayBase(Recycler, recycler, AllocLeafTrace, T, count) #define RecyclerNewFinalizedTrace(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedTrace, T, __VA_ARGS__))) #define RecyclerNewFinalizedLeafTrace(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocFinalizedLeafTrace, T, __VA_ARGS__))) #define RecyclerNewFinalizedPlusTrace(recycler, size, T,...) static_cast(static_cast(AllocatorNewPlusBase(Recycler, recycler, AllocFinalizedTrace, size, T, __VA_ARGS__))) #define RecyclerNewTrackedTrace(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedTrace, T, __VA_ARGS__))) #define RecyclerNewTrackedLeafTrace(recycler,T,...) static_cast(static_cast(AllocatorNewBase(Recycler, recycler, AllocTrackedLeafTrace, T, __VA_ARGS__))) #else #define RecyclerNewLeafTrace RecyclerNewLeaf #define RecyclerNewLeafZTrace RecyclerNewLeafZ #define RecyclerNewPlusLeafTrace RecyclerNewPlusLeaf #define RecyclerNewArrayLeafZTrace RecyclerNewArrayLeafZ #define RecyclerNewArrayTrace RecyclerNewArray #define RecyclerNewArrayZTrace RecyclerNewArrayZ #define RecyclerNewArrayLeafTrace RecyclerNewArrayLeaf #define RecyclerNewFinalizedTrace RecyclerNewFinalized #define RecyclerNewFinalizedLeafTrace RecyclerNewFinalizedLeaf #define RecyclerNewFinalizedPlusTrace RecyclerNewFinalizedPlus #define RecyclerNewTrackedTrace RecyclerNewTracked #define RecyclerNewTrackedLeafTrace RecyclerNewTrackedLeaf #endif #ifdef RECYCLER_TRACE #define RecyclerVerboseTrace(flags, ...) \ if (flags.Verbose && flags.Trace.IsEnabled(Js::RecyclerPhase)) \ { \ Output::Print(__VA_ARGS__); \ } #define AllocationVerboseTrace(flags, ...) \ if (flags.Verbose && flags.Trace.IsEnabled(Js::MemoryAllocationPhase)) \ { \ Output::Print(__VA_ARGS__); \ } #define LargeAllocationVerboseTrace(flags, ...) \ if (flags.Verbose && \ (flags.Trace.IsEnabled(Js::MemoryAllocationPhase) || \ flags.Trace.IsEnabled(Js::LargeMemoryAllocationPhase))) \ { \ Output::Print(__VA_ARGS__); \ } #define PageAllocatorAllocationVerboseTrace(flags, ...) \ if (flags.Verbose && flags.Trace.IsEnabled(Js::PageAllocatorAllocPhase)) \ { \ Output::Print(__VA_ARGS__); \ } #else #define RecyclerVerboseTrace(...) #define AllocationVerboseTrace(...) #define LargeAllocationVerboseTrace(...) #endif #define RecyclerHeapNew(recycler,heapInfo,T,...) new (recycler, heapInfo) T(__VA_ARGS__) #define RecyclerHeapDelete(recycler,heapInfo,addr) (static_cast(recycler)->HeapFree(heapInfo,addr)) typedef void (__cdecl* ExternalRootMarker)(void *); class RecyclerCollectionWrapper { public: RecyclerCollectionWrapper() : _isScriptContextCloseGCPending(FALSE) { } typedef BOOL (Recycler::*CollectionFunction)(CollectionFlags flags); virtual void PreCollectionCallBack(CollectionFlags flags) = 0; virtual void PreSweepCallback() = 0; virtual void PreRescanMarkCallback() = 0; virtual size_t RootMarkCallback(RecyclerScanMemoryCallback& scanMemoryCallback, BOOL * stacksScannedByRuntime) = 0; virtual void RescanMarkTimeoutCallback() = 0; virtual void EndMarkCallback() = 0; virtual void ConcurrentCallback() = 0; virtual void WaitCollectionCallBack() = 0; virtual void PostCollectionCallBack() = 0; virtual BOOL ExecuteRecyclerCollectionFunction(Recycler * recycler, CollectionFunction function, CollectionFlags flags) = 0; virtual uint GetRandomNumber() = 0; virtual bool DoSpecialMarkOnScanStack() = 0; virtual void OnScanStackCallback(void ** stackTop, size_t byteCount, void ** registers, size_t registersByteCount) = 0; virtual void PostSweepRedeferralCallBack() = 0; #ifdef FAULT_INJECTION virtual void DisposeScriptContextByFaultInjectionCallBack() = 0; #endif virtual void DisposeObjects(Recycler * recycler) = 0; virtual void PreDisposeObjectsCallBack() = 0; #ifdef ENABLE_PROJECTION virtual void MarkExternalWeakReferencedObjects(bool inPartialCollect) = 0; virtual void ResolveExternalWeakReferencedObjects() = 0; #endif #if DBG || defined(PROFILE_EXEC) virtual bool AsyncHostOperationStart(void *) = 0; virtual void AsyncHostOperationEnd(bool wasInAsync, void *) = 0; #endif #if DBG virtual void CheckJsReentrancyOnDispose() = 0; #endif BOOL GetIsScriptContextCloseGCPending() { return _isScriptContextCloseGCPending; } void ClearIsScriptContextCloseGCPending() { _isScriptContextCloseGCPending = FALSE; } void SetIsScriptContextCloseGCPending() { _isScriptContextCloseGCPending = TRUE; } protected: BOOL _isScriptContextCloseGCPending; }; class DefaultRecyclerCollectionWrapper : public RecyclerCollectionWrapper { public: virtual void PreCollectionCallBack(CollectionFlags flags) override {} virtual void PreSweepCallback() override {} virtual void PreRescanMarkCallback() override {} virtual void RescanMarkTimeoutCallback() override {} virtual void EndMarkCallback() override {} virtual size_t RootMarkCallback(RecyclerScanMemoryCallback& scanMemoryCallback, BOOL * stacksScannedByRuntime) override { *stacksScannedByRuntime = FALSE; return 0; } virtual void ConcurrentCallback() override {} virtual void WaitCollectionCallBack() override {} virtual void PostCollectionCallBack() override {} virtual BOOL ExecuteRecyclerCollectionFunction(Recycler * recycler, CollectionFunction function, CollectionFlags flags) override; virtual uint GetRandomNumber() override { return 0; } virtual bool DoSpecialMarkOnScanStack() override { return false; } virtual void OnScanStackCallback(void ** stackTop, size_t byteCount, void ** registers, size_t registersByteCount) override {}; virtual void PostSweepRedeferralCallBack() override {} #ifdef FAULT_INJECTION virtual void DisposeScriptContextByFaultInjectionCallBack() override {}; #endif virtual void DisposeObjects(Recycler * recycler) override; virtual void PreDisposeObjectsCallBack() override {}; #ifdef ENABLE_PROJECTION virtual void MarkExternalWeakReferencedObjects(bool inPartialCollect) override {}; virtual void ResolveExternalWeakReferencedObjects() override {}; #endif #if DBG || defined(PROFILE_EXEC) virtual bool AsyncHostOperationStart(void *) override { return false; }; virtual void AsyncHostOperationEnd(bool wasInAsync, void *) override {}; #endif #if DBG virtual void CheckJsReentrancyOnDispose() override {} #endif static DefaultRecyclerCollectionWrapper Instance; private: static bool IsCollectionDisabled(Recycler * recycler); }; #ifdef RECYCLER_STATS struct RecyclerCollectionStats { size_t startCollectAllocBytes; #if ENABLE_PARTIAL_GC size_t startCollectNewPageCount; #endif size_t continueCollectAllocBytes; size_t finishCollectTryCount; // Heuristic Stats #if ENABLE_PARTIAL_GC size_t rescanRootBytes; size_t estimatedPartialReuseBytes; size_t uncollectedNewPageCountPartialCollect; size_t partialCollectSmallHeapBlockReuseMinFreeBytes; double collectEfficacy; double collectCost; #endif // Mark stats size_t tryMarkCount; // # of pointer try mark (* pointer size to get total number byte looked at) size_t tryMarkNullCount; size_t tryMarkUnalignedCount; size_t tryMarkNonRecyclerMemoryCount; size_t tryMarkInteriorCount; size_t tryMarkInteriorNullCount; size_t tryMarkInteriorNonRecyclerMemoryCount; size_t rootCount; size_t stackCount; size_t remarkCount; size_t scanCount; // non-leaf objects marked. size_t trackCount; size_t finalizeCount; size_t markThruNewObjCount; size_t markThruFalseNewObjCount; struct MarkData { // Rescan stats size_t rescanPageCount; size_t rescanObjectCount; size_t rescanObjectByteCount; size_t rescanLargePageCount; size_t rescanLargeObjectCount; size_t rescanLargeByteCount; size_t markCount; // total number of object marked size_t markBytes; // size of all objects marked. } markData; #if ENABLE_CONCURRENT_GC MarkData backgroundMarkData[RecyclerHeuristic::MaxBackgroundRepeatMarkCount]; size_t trackedObjectCount; #endif #if ENABLE_PARTIAL_GC size_t clientTrackedObjectCount; #endif // Sweep stats size_t heapBlockCount[HeapBlock::BlockTypeCount]; // number of heap blocks (processed during swept) size_t heapBlockFreeCount[HeapBlock::BlockTypeCount]; // number of heap blocks deleted size_t heapBlockConcurrentSweptCount[HeapBlock::SmallBlockTypeCount]; size_t heapBlockSweptCount[HeapBlock::SmallBlockTypeCount]; // number of heap blocks swept size_t objectSweptCount; // objects freed (free list + whole page freed) size_t objectSweptBytes; size_t objectSweptFreeListCount; // objects freed (free list) size_t objectSweptFreeListBytes; size_t objectSweepScanCount; // number of objects walked for sweeping (exclude whole page freed) size_t finalizeSweepCount; // number of objects finalizer/dispose called #if ENABLE_PARTIAL_GC size_t smallNonLeafHeapBlockPartialReuseCount[HeapBlock::SmallBlockTypeCount]; size_t smallNonLeafHeapBlockPartialReuseBytes[HeapBlock::SmallBlockTypeCount]; size_t smallNonLeafHeapBlockPartialUnusedCount[HeapBlock::SmallBlockTypeCount]; size_t smallNonLeafHeapBlockPartialUnusedBytes[HeapBlock::SmallBlockTypeCount]; #endif // Memory Stats size_t heapBlockFreeByteCount[HeapBlock::BlockTypeCount]; // The remaining usable free byte count size_t largeHeapBlockUsedByteCount; // Used byte count size_t largeHeapBlockTotalByteCount; // Total byte count // Empty/zero heap block stats uint numEmptySmallBlocks[HeapBlock::SmallBlockTypeCount]; uint numZeroedOutSmallBlocks; }; #define RECYCLER_STATS_INC_IF(cond, r, f) if (cond) { RECYCLER_STATS_INC(r, f); } #define RECYCLER_STATS_INC(r, f) ++r->collectionStats.f #define RECYCLER_STATS_INTERLOCKED_INC(r, f) { InterlockedIncrement((LONG *)&r->collectionStats.f); } #define RECYCLER_STATS_DEC(r, f) --r->collectionStats.f #define RECYCLER_STATS_ADD(r, f, v) r->collectionStats.f += (v) #define RECYCLER_STATS_INTERLOCKED_ADD(r, f, v) { InterlockedAdd((LONG *)&r->collectionStats.f, (LONG)(v)); } #define RECYCLER_STATS_SUB(r, f, v) r->collectionStats.f -= (v) #define RECYCLER_STATS_SET(r, f, v) r->collectionStats.f = v #else #define RECYCLER_STATS_INC_IF(cond, r, f) #define RECYCLER_STATS_INC(r, f) #define RECYCLER_STATS_INTERLOCKED_INC(r, f) #define RECYCLER_STATS_DEC(r, f) #define RECYCLER_STATS_ADD(r, f, v) #define RECYCLER_STATS_INTERLOCKED_ADD(r, f, v) #define RECYCLER_STATS_SUB(r, f, v) #define RECYCLER_STATS_SET(r, f, v) #endif #ifdef RECYCLER_TRACE struct CollectionParam { CollectionFlags flags; bool finishOnly; bool repeat; bool priorityBoostConcurrentSweepOverride; bool domCollect; int timeDiff; size_t uncollectedAllocBytes; size_t uncollectedPinnedObjects; #if ENABLE_PARTIAL_GC size_t uncollectedNewPageCountPartialCollect; size_t uncollectedNewPageCount; size_t unusedPartialCollectFreeBytes; bool inPartialCollectMode; #endif }; #endif #include "RecyclerObjectGraphDumper.h" #if ENABLE_CONCURRENT_GC class RecyclerParallelThread { friend class ThreadContext; public: typedef void (Recycler::* WorkFunc)(); RecyclerParallelThread(Recycler * recycler, WorkFunc workFunc) : recycler(recycler), workFunc(workFunc), concurrentWorkReadyEvent(NULL), concurrentWorkDoneEvent(NULL), concurrentThread(NULL) { } ~RecyclerParallelThread() { Assert(concurrentThread == NULL); Assert(concurrentWorkReadyEvent == NULL); Assert(concurrentWorkDoneEvent == NULL); } bool StartConcurrent(); void WaitForConcurrent(); void Shutdown(); bool EnableConcurrent(bool synchronizeOnStartup); private: // Static entry point for thread creation static unsigned int CALLBACK StaticThreadProc(LPVOID lpParameter); // Static entry point for thread service usage static void CALLBACK StaticBackgroundWorkCallback(void * callbackData); private: WorkFunc workFunc; Recycler * recycler; HANDLE concurrentWorkReadyEvent;// main thread uses this event to tell concurrent threads that the work is ready HANDLE concurrentWorkDoneEvent;// concurrent threads use this event to tell main thread that the work allocated is done HANDLE concurrentThread; bool synchronizeOnStartup; }; #endif #ifdef ENABLE_DEBUG_CONFIG_OPTIONS class AutoProtectPages { public: AutoProtectPages(Recycler* recycler, bool protectEnabled); ~AutoProtectPages(); void Unprotect(); private: Recycler* recycler; bool isReadOnly; }; #endif class Recycler { friend class RecyclerScanMemoryCallback; friend class RecyclerSweep; friend class RecyclerSweepManager; friend class MarkContext; friend class HeapBlock; friend class HeapBlockMap32; #if ENABLE_CONCURRENT_GC friend class RecyclerParallelThread; #endif #ifdef ENABLE_DEBUG_CONFIG_OPTIONS friend class AutoProtectPages; #endif #ifdef ENABLE_BASIC_TELEMETRY friend class RecyclerTelemetryInfo; #endif template friend class RecyclerWeakReference; template friend class WeakReferenceHashTable; template friend class SmallHeapBlockAllocator; // Needed for FindHeapBlock #if defined(RECYCLER_TRACE) friend class JavascriptThreadService; #endif #ifdef HEAP_ENUMERATION_VALIDATION friend class ActiveScriptProfilerHeapEnum; #endif friend class ScriptEngineBase; // This is for disabling GC for certain Host operations. #if !FLOATVAR friend class ::CodeGenNumberThreadAllocator; friend struct ::XProcNumberPageSegmentManager; #endif public: static const uint ConcurrentThreadStackSize = 300000; static const bool FakeZeroLengthArray = true; #ifdef RECYCLER_PAGE_HEAP // Keeping as constant in case we want to tweak the value here // Set to 0 so that the tool can do the filtering instead of the runtime #if DBG static const int s_numFramesToSkipForPageHeapAlloc = 10; static const int s_numFramesToSkipForPageHeapFree = 0; static const int s_numFramesToCaptureForPageHeap = 32; #else static const int s_numFramesToSkipForPageHeapAlloc = 0; static const int s_numFramesToSkipForPageHeapFree = 0; static const int s_numFramesToCaptureForPageHeap = 32; #endif #endif uint Cookie; class AutoEnterExternalStackSkippingGCMode { public: AutoEnterExternalStackSkippingGCMode(Recycler* recycler): _recycler(recycler) { // Setting this in a re-entrant mode is not allowed Assert(!recycler->isExternalStackSkippingGC); #if DBG _recycler->isExternalStackSkippingGC = true; #endif } ~AutoEnterExternalStackSkippingGCMode() { #if DBG _recycler->isExternalStackSkippingGC = false; #endif } private: Recycler* _recycler; }; private: class AutoSwitchCollectionStates { public: AutoSwitchCollectionStates(Recycler* recycler, CollectionState entryState, CollectionState exitState): _recycler(recycler), _exitState(exitState) { _recycler->SetCollectionState(entryState); } ~AutoSwitchCollectionStates() { _recycler->SetCollectionState(_exitState); } private: Recycler* _recycler; CollectionState _exitState; }; #if defined(ENABLE_JS_ETW) ETWEventGCActivationTrigger collectionStartReason; CollectionFlags collectionStartFlags; ETWEventGCActivationTrigger collectionFinishReason; #endif class CollectionStateChangedObserver : public ObservableValueObserver { private: Recycler* recycler; public: CollectionStateChangedObserver(Recycler* recycler) { this->recycler = recycler; } virtual void ValueChanged(const CollectionState& newVal, const CollectionState& oldVal) { #ifdef ENABLE_BASIC_TELEMETRY if (oldVal == CollectionState::CollectionStateNotCollecting && newVal != CollectionState::CollectionStateNotCollecting && newVal != CollectionState::Collection_PreCollection && newVal != CollectionState::CollectionStateExit) { this->recycler->GetRecyclerTelemetryInfo().StartPass(newVal); } else if (oldVal != CollectionState::CollectionStateNotCollecting && oldVal != CollectionState::Collection_PreCollection && oldVal != CollectionState::CollectionStateExit && newVal == CollectionState::CollectionStateNotCollecting) { this->recycler->GetRecyclerTelemetryInfo().EndPass(oldVal); } #endif } }; CollectionStateChangedObserver collectionStateChangedObserver; ObservableValue collectionState; inline void SetCollectionState(CollectionState newState) { this->collectionState = newState; } JsUtil::ThreadService *threadService; #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP bool allowAllocationsDuringConcurrentSweepForCollection; #endif HeapBlockMap heapBlockMap; #if defined(CHECK_MEMORY_LEAK) || defined(LEAK_REPORT) struct PinRecord { #ifdef STACK_BACK_TRACE PinRecord() : refCount(0), stackBackTraces(nullptr) {} #else PinRecord() : refCount(0) {} #endif PinRecord& operator=(uint newRefCount) { #ifdef STACK_BACK_TRACE Assert(stackBackTraces == nullptr); #endif Assert(newRefCount == 0); refCount = 0; return *this; } PinRecord& operator++() { ++refCount; return *this; } PinRecord& operator--() { --refCount; return *this; } operator uint() const { return refCount; } #ifdef STACK_BACK_TRACE StackBackTraceNode * stackBackTraces; #endif private: uint refCount; }; #else typedef uint PinRecord; #endif typedef SimpleHashTable PinnedObjectHashTable; PinnedObjectHashTable pinnedObjectMap; WeakReferenceHashTable weakReferenceMap; uint weakReferenceCleanupId; #if ENABLE_WEAK_REFERENCE_REGIONS SList weakReferenceRegionList; #endif void * transientPinnedObject; #if defined(CHECK_MEMORY_LEAK) || defined(LEAK_REPORT) #ifdef STACK_BACK_TRACE StackBackTrace * transientPinnedObjectStackBackTrace; #endif #endif struct GuestArenaAllocator : public ArenaAllocator { GuestArenaAllocator(__in_z char16 const* name, PageAllocator * pageAllocator, void (*outOfMemoryFunc)()) : ArenaAllocator(name, pageAllocator, outOfMemoryFunc), pendingDelete(false) { } bool pendingDelete; }; DListBase guestArenaList; DListBase externalGuestArenaList; // guest arenas are scanned for roots #ifdef RECYCLER_PAGE_HEAP bool isPageHeapEnabled; bool capturePageHeapAllocStack; bool capturePageHeapFreeStack; inline bool IsPageHeapEnabled() const { return isPageHeapEnabled; } inline bool ShouldCapturePageHeapAllocStack() const { return capturePageHeapAllocStack; } void VerifyPageHeapFillAfterAlloc(char* memBlock, size_t size, ObjectInfoBits attributes); #else inline const bool IsPageHeapEnabled() const { return false; } inline bool ShouldCapturePageHeapAllocStack() const { return false; } #endif #ifdef RECYCLER_MARK_TRACK MarkMap* markMap; CriticalSection markMapCriticalSection; void PrintMarkMap(); void ClearMarkMap(); #endif // Number of pages to reserve for the primary mark stack // This is the minimum number of pages to guarantee that a single heap block // can be rescanned in the worst possible case where every object in a heap block // in the smallest bucket needs to be rescanned // These many pages being reserved guarantees that in OOM Rescan, we can make progress // on every rescan iteration // We add one because there is a small amount of the page reserved for page pool metadata // so we need to allocate an additional page to be sure // Currently, this works out to 2 pages on 32-bit and 5 pages on 64-bit // NOTE: We have reduced the PageCount for small blocks to 1. This brought down the number of pages reserved for x64 from 5 to 2. This has not shown // any adverse impact. static const int PrimaryMarkStackReservedPageCount = ((SmallAllocationBlockAttributes::PageCount * MarkContext::MarkCandidateSize) / SmallAllocationBlockAttributes::MinObjectSize) + 1; MarkContext markContext; // Contexts for parallel marking. // We support up to 4 way parallelism, main context + 3 additional parallel contexts. MarkContext parallelMarkContext1; MarkContext parallelMarkContext2; MarkContext parallelMarkContext3; // Page pools for above markContexts PagePool markPagePool; PagePool parallelMarkPagePool1; PagePool parallelMarkPagePool2; PagePool parallelMarkPagePool3; bool IsMarkStackEmpty(); bool HasPendingMarkObjects() const { return markContext.HasPendingMarkObjects() || parallelMarkContext1.HasPendingMarkObjects() || parallelMarkContext2.HasPendingMarkObjects() || parallelMarkContext3.HasPendingMarkObjects(); } bool HasPendingTrackObjects() const { return markContext.HasPendingTrackObjects() || parallelMarkContext1.HasPendingTrackObjects() || parallelMarkContext2.HasPendingTrackObjects() || parallelMarkContext3.HasPendingTrackObjects(); } RecyclerCollectionWrapper * collectionWrapper; HANDLE mainThreadHandle; void * stackBase; class SavedRegisterState { public: #if _M_IX86 static const int NumRegistersToSave = 8; #elif _M_ARM static const int NumRegistersToSave = 13; #elif _M_ARM64 static const int NumRegistersToSave = 27; #elif _M_AMD64 static const int NumRegistersToSave = 16; #endif SavedRegisterState() { memset(registers, 0, sizeof(void*) * NumRegistersToSave); } void** GetRegisters() { return registers; } void* GetStackTop() { // By convention, our register-saving routine will always // save the stack pointer as the first item in the array return registers[0]; } private: void* registers[NumRegistersToSave]; }; SavedRegisterState savedThreadContext; #if __has_feature(address_sanitizer) void* savedAsanFakeStack; #define SAVE_THREAD_ASAN_FAKE_STACK() \ this->savedAsanFakeStack = __asan_get_current_fake_stack() #else #define SAVE_THREAD_ASAN_FAKE_STACK() #endif bool inDispose; #if DBG || defined RECYCLER_TRACE uint collectionCount; bool inResolveExternalWeakReferences; #endif bool allowDispose; bool inDisposeWrapper; bool needOOMRescan; bool hasDisposableObject; bool hasNativeGCHost; DWORD tickCountNextDispose; bool inExhaustiveCollection; bool hasExhaustiveCandidate; bool inCacheCleanupCollection; bool inDecommitNowCollection; bool isScriptActive; bool isInScript; bool isShuttingDown; bool scanPinnedObjectMap; bool hasScannedInitialImplicitRoots; bool hasPendingUnpinnedObject; bool hasPendingDeleteGuestArena; bool inEndMarkOnLowMemory; bool decommitOnFinish; bool enableScanInteriorPointers; bool enableScanImplicitRoots; bool disableCollectOnAllocationHeuristics; #ifdef ENABLE_DEBUG_CONFIG_OPTIONS bool disableCollection; #endif #if ENABLE_PARTIAL_GC bool enablePartialCollect; bool inPartialCollectMode; #if ENABLE_CONCURRENT_GC bool hasBackgroundFinishPartial; bool partialConcurrentNextCollection; #endif #endif #ifdef RECYCLER_STRESS bool forcePartialScanStack; bool recyclerStress; #if ENABLE_CONCURRENT_GC bool recyclerBackgroundStress; bool recyclerConcurrentStress; bool recyclerConcurrentRepeatStress; #endif #if ENABLE_PARTIAL_GC bool recyclerPartialStress; #endif #endif #if DBG bool isExternalStackSkippingGC; #endif bool skipStack; #if ENABLE_CONCURRENT_GC #if DBG bool isConcurrentGCOnIdle; bool isFinishGCOnIdle; #endif bool queueTrackedObject; bool hasPendingConcurrentFindRoot; bool priorityBoost; bool disableConcurrent; bool enableConcurrentMark; bool enableParallelMark; bool enableConcurrentSweep; uint maxParallelism; // Max # of total threads to run in parallel byte backgroundRescanCount; // for ETW events and stats byte backgroundFinishMarkCount; size_t backgroundRescanRootBytes; HANDLE concurrentWorkReadyEvent; // main thread uses this event to tell concurrent threads that the work is ready HANDLE concurrentWorkDoneEvent; // concurrent threads use this event to tell main thread that the work allocated is done HANDLE concurrentThread; template void ParallelWorkFunc(); RecyclerParallelThread parallelThread1; RecyclerParallelThread parallelThread2; #if DBG // Variable indicating if the concurrent thread has exited or not // If the concurrent thread hasn't started yet, this is set to true // Once the concurrent thread starts, it sets this to false, // and when the concurrent thread exits, it sets this to true. bool concurrentThreadExited; bool disableConcurrentThreadExitedCheck; bool isProcessingTrackedObjects; #endif uint tickCountStartConcurrent; bool isAborting; #endif #if DBG bool hasIncompleteDoCollect; // This is set to true when we begin a Rescan, and set to false when either: // (1) We finish the final in-thread Rescan and are about to Mark // (2) We do a conditional ResetWriteWatch and are about to Mark // When this flag is true, we should not be modifying existing mark-related state, // including markBits and rescanState. bool isProcessingRescan; #endif Js::ConfigFlagsTable& recyclerFlagsTable; RecyclerSweepManager recyclerSweepManagerInstance; RecyclerSweepManager * recyclerSweepManager; static const uint tickDiffToNextCollect = 300; #ifdef IDLE_DECOMMIT_ENABLED HANDLE concurrentIdleDecommitEvent; LONG needIdleDecommitSignal; #endif #if ENABLE_PARTIAL_GC SListBase clientTrackedObjectList; ArenaAllocator clientTrackedObjectAllocator; size_t partialUncollectedAllocBytes; // Dynamic Heuristics for partial GC size_t uncollectedNewPageCountPartialCollect; #endif uint tickCountNextCollection; uint tickCountNextFinishCollection; void (*outOfMemoryFunc)(); #ifdef RECYCLER_TEST_SUPPORT BOOL (*checkFn)(char* addr, size_t size); #endif ExternalRootMarker externalRootMarker; void * externalRootMarkerContext; #ifdef PROFILE_EXEC Js::Profiler * profiler; Js::Profiler * backgroundProfiler; PageAllocator backgroundProfilerPageAllocator; DListBase backgroundProfilerArena; #endif // destruct autoHeap after backgroundProfilerPageAllocator; HeapInfoManager autoHeap; template HeapInfo * GetHeapInfoForAllocation() { return this->GetHeapInfo(); } template HeapInfo * GetHeapInfo() { return this->autoHeap.GetDefaultHeap(); } HeapInfo * GetHeapInfo() { return this->autoHeap.GetDefaultHeap(); } #ifdef PROFILE_MEM RecyclerMemoryData * memoryData; #endif ThreadContextId mainThreadId; #if DBG uint heapBlockCount; bool disableThreadAccessCheck; #endif #if DBG || defined(RECYCLER_STATS) bool isForceSweeping; #endif #ifdef NTBUILD RecyclerWatsonTelemetryBlock localTelemetryBlock; RecyclerWatsonTelemetryBlock * telemetryBlock; #endif #ifdef ENABLE_BASIC_TELEMETRY private: RecyclerTelemetryInfo telemetryStats; GUID recyclerID; public: GUID& GetRecyclerID() { return this->recyclerID; } #endif public: bool GetIsInScript() { return this->isInScript; } bool GetIsScriptActive() { return this->isScriptActive; } private: #ifdef RECYCLER_STATS RecyclerCollectionStats collectionStats; void PrintHeapBlockStats(char16 const * name, HeapBlock::HeapBlockType type); void PrintHeapBlockMemoryStats(char16 const * name, HeapBlock::HeapBlockType type); void PrintCollectStats(); void PrintHeuristicCollectionStats(); void PrintMarkCollectionStats(); void PrintBackgroundCollectionStats(); void PrintMemoryStats(); void PrintBackgroundCollectionStat(RecyclerCollectionStats::MarkData const& markData); #endif #ifdef RECYCLER_TRACE CollectionParam collectionParam; void PrintBlockStatus(HeapBucket * heapBucket, HeapBlock * heapBlock, char16 const * name); #endif #ifdef RECYCLER_MEMORY_VERIFY uint verifyPad; bool verifyEnabled; #endif #ifdef RECYCLER_DUMP_OBJECT_GRAPH friend class RecyclerObjectGraphDumper; RecyclerObjectGraphDumper * objectGraphDumper; public: bool dumpObjectOnceOnCollect; #endif public: Recycler(AllocationPolicyManager * policyManager, IdleDecommitPageAllocator * pageAllocator, void(*outOfMemoryFunc)(), Js::ConfigFlagsTable& flags, RecyclerTelemetryHostInterface* hostInterface); ~Recycler(); void Initialize(const bool forceInThread, JsUtil::ThreadService *threadService, const bool deferThreadStartup = false #ifdef RECYCLER_PAGE_HEAP , PageHeapMode pageheapmode = PageHeapMode::PageHeapModeOff , bool captureAllocCallStack = false , bool captureFreeCallStack = false #endif ); Js::ConfigFlagsTable& GetRecyclerFlagsTable() const { return this->recyclerFlagsTable; } void SetMemProtectMode(); bool IsMemProtectMode(); size_t GetUsedBytes(); void LogMemProtectHeapSize(bool fromGC); char* Realloc(void* buffer, DECLSPEC_GUARD_OVERFLOW size_t existingBytes, DECLSPEC_GUARD_OVERFLOW size_t requestedBytes, bool truncate = true); #ifdef NTBUILD void SetTelemetryBlock(RecyclerWatsonTelemetryBlock * telemetryBlock) { this->telemetryBlock = telemetryBlock; } #endif uint GetPinnedObjectCount() const { return this->pinnedObjectMap.Count(); } void Prime(); void* GetOwnerContext() { return (void*) this->collectionWrapper; } bool NeedOOMRescan() const; void SetNeedOOMRescan(); void ClearNeedOOMRescan(); BOOL RequestConcurrentWrapperCallback(); BOOL CollectionInProgress() const; BOOL IsExiting() const; BOOL IsSweeping() const; #ifdef RECYCLER_PAGE_HEAP inline bool ShouldCapturePageHeapFreeStack() const { return capturePageHeapFreeStack; } #else inline bool ShouldCapturePageHeapFreeStack() const { return false; } #endif void SetIsThreadBound(); void SetIsScriptActive(bool isScriptActive); void SetIsInScript(bool isInScript); bool HasNativeGCHost() const; void SetHasNativeGCHost(); bool ShouldIdleCollectOnExit(); void ScheduleNextCollection(); BOOL IsShuttingDown() const { return this->isShuttingDown; } #if ENABLE_CONCURRENT_GC #if DBG BOOL IsConcurrentMarkEnabled() const { return enableConcurrentMark; } BOOL IsConcurrentSweepEnabled() const { return enableConcurrentSweep; } #endif template BOOL FinishConcurrent(); void ShutdownThread(); bool EnableConcurrent(JsUtil::ThreadService *threadService, bool startAllThreads); void DisableConcurrent(); void StartQueueTrackedObject(); bool DoQueueTrackedObject() const; void PrepareSweep(); #endif template void SetupPostCollectionFlags(); void EnsureNotCollecting(); #if ENABLE_CONCURRENT_GC bool QueueTrackedObject(FinalizableObject * trackableObject); #endif // FindRoots void TryMarkNonInterior(void* candidate, void* parentReference = nullptr); void TryMarkInterior(void *candidate, void* parentReference = nullptr); bool InCacheCleanupCollection() { return inCacheCleanupCollection; } void ClearCacheCleanupCollection() { Assert(inCacheCleanupCollection); inCacheCleanupCollection = false; } // Finalizer support void SetExternalRootMarker(ExternalRootMarker fn, void * context); ArenaAllocator * CreateGuestArena(char16 const * name, void (*outOfMemoryFunc)()); void DeleteGuestArena(ArenaAllocator * arenaAllocator); ArenaData ** RegisterExternalGuestArena(ArenaData* guestArena) { return externalGuestArenaList.PrependNode(&NoThrowHeapAllocator::Instance, guestArena); } void UnregisterExternalGuestArena(ArenaData* guestArena) { externalGuestArenaList.Remove(&NoThrowHeapAllocator::Instance, guestArena); // Any time a root is removed during a GC, it indicates that an exhaustive // collection is likely going to have work to do so trigger an exhaustive // candidate GC to indicate this fact this->CollectNow(); } void UnregisterExternalGuestArena(ArenaData** guestArena) { externalGuestArenaList.RemoveElement(&NoThrowHeapAllocator::Instance, guestArena); // Any time a root is removed during a GC, it indicates that an exhaustive // collection is likely going to have work to do so trigger an exhaustive // candidate GC to indicate this fact this->CollectNow(); } #ifdef RECYCLER_TEST_SUPPORT void SetCheckFn(BOOL(*checkFn)(char* addr, size_t size)); #endif void SetCollectionWrapper(RecyclerCollectionWrapper * wrapper); static size_t GetAlignedSize(size_t size) { return HeapInfo::GetAlignedSize(size); } HeapInfo* GetDefaultHeapInfo() { return autoHeap.GetDefaultHeap(); } template BOOL CollectNow(); #ifdef ENABLE_DEBUG_CONFIG_OPTIONS void DisplayMemStats(); #endif void AddExternalMemoryUsage(size_t size); bool NeedDispose() { return this->hasDisposableObject; } template bool FinishDisposeObjectsNow(); bool RequestExternalMemoryAllocation(size_t size); void ReportExternalMemoryFailure(size_t size); void ReportExternalMemoryFree(size_t size); // ExternalAllocFunc returns true when allocation succeeds template bool DoExternalAllocation(size_t size, ExternalAllocFunc externalAllocFunc); #ifdef TRACE_OBJECT_LIFETIME #define DEFINE_RECYCLER_ALLOC_TRACE(AllocFunc, AllocWithAttributesFunc, attributes) \ inline char* AllocFunc##Trace(size_t size) \ { \ return AllocWithAttributesFunc<(ObjectInfoBits)(attributes | TraceBit), /* nothrow = */ false>(size); \ } #else #define DEFINE_RECYCLER_ALLOC_TRACE(AllocFunc, AllocWithAttributeFunc, attributes) #endif #define DEFINE_RECYCLER_ALLOC_BASE(AllocFunc, AllocWithAttributesFunc, attributes) \ inline char * AllocFunc(DECLSPEC_GUARD_OVERFLOW size_t size) \ { \ return AllocWithAttributesFunc(size); \ } \ __forceinline char * AllocFunc##Inlined(DECLSPEC_GUARD_OVERFLOW size_t size) \ { \ return AllocWithAttributesFunc##Inlined(size); \ } \ DEFINE_RECYCLER_ALLOC_TRACE(AllocFunc, AllocWithAttributesFunc, attributes); #define DEFINE_RECYCLER_NOTHROW_ALLOC_BASE(AllocFunc, AllocWithAttributesFunc, attributes) \ inline char * NoThrow##AllocFunc(DECLSPEC_GUARD_OVERFLOW size_t size) \ { \ return AllocWithAttributesFunc(size); \ } \ inline char * NoThrow##AllocFunc##Inlined(DECLSPEC_GUARD_OVERFLOW size_t size) \ { \ return AllocWithAttributesFunc##Inlined(size); \ } \ DEFINE_RECYCLER_ALLOC_TRACE(AllocFunc, AllocWithAttributesFunc, attributes); #define DEFINE_RECYCLER_ALLOC(AllocFunc, attributes) DEFINE_RECYCLER_ALLOC_BASE(AllocFunc, AllocWithAttributes, attributes) #define DEFINE_RECYCLER_ALLOC_ZERO(AllocFunc, attributes) DEFINE_RECYCLER_ALLOC_BASE(AllocFunc, AllocZeroWithAttributes, attributes) #define DEFINE_RECYCLER_NOTHROW_ALLOC(AllocFunc, attributes) DEFINE_RECYCLER_NOTHROW_ALLOC_BASE(AllocFunc, AllocWithAttributes, attributes) #define DEFINE_RECYCLER_NOTHROW_ALLOC_ZERO(AllocFunc, attributes) DEFINE_RECYCLER_NOTHROW_ALLOC_BASE(AllocFunc, AllocZeroWithAttributes, attributes) #if GLOBAL_ENABLE_WRITE_BARRIER DEFINE_RECYCLER_ALLOC(Alloc, WithBarrierBit); DEFINE_RECYCLER_ALLOC_ZERO(AllocZero, WithBarrierBit); DEFINE_RECYCLER_ALLOC(AllocFinalized, FinalizableWithBarrierObjectBits); DEFINE_RECYCLER_ALLOC(AllocTracked, ClientTrackableObjectWithBarrierBits); DEFINE_RECYCLER_ALLOC(AllocFinalizedClientTracked, ClientTrackableObjectWithBarrierBits); #else DEFINE_RECYCLER_ALLOC(Alloc, NoBit); DEFINE_RECYCLER_ALLOC_ZERO(AllocZero, NoBit); DEFINE_RECYCLER_ALLOC(AllocFinalized, FinalizableObjectBits); DEFINE_RECYCLER_ALLOC(AllocTracked, ClientTrackableObjectBits); DEFINE_RECYCLER_ALLOC(AllocFinalizedClientTracked, ClientFinalizableObjectBits); #endif #ifdef RECYCLER_WRITE_BARRIER_ALLOC DEFINE_RECYCLER_ALLOC(AllocWithBarrier, WithBarrierBit); DEFINE_RECYCLER_ALLOC_ZERO(AllocZeroWithBarrier, WithBarrierBit); DEFINE_RECYCLER_ALLOC(AllocFinalizedWithBarrier, FinalizableWithBarrierObjectBits); DEFINE_RECYCLER_ALLOC(AllocTrackedWithBarrier, ClientTrackableObjectWithBarrierBits); DEFINE_RECYCLER_ALLOC(AllocFinalizedClientTrackedWithBarrier, ClientFinalizableObjectWithBarrierBits); #endif DEFINE_RECYCLER_ALLOC(AllocLeaf, LeafBit); DEFINE_RECYCLER_ALLOC(AllocFinalizedLeaf, FinalizableLeafBits); DEFINE_RECYCLER_ALLOC(AllocTrackedLeaf, ClientTrackableLeafBits); DEFINE_RECYCLER_ALLOC_ZERO(AllocLeafZero, LeafBit); DEFINE_RECYCLER_ALLOC_ZERO(AllocZeroTrackedLeaf, ClientTrackableLeafBits); DEFINE_RECYCLER_NOTHROW_ALLOC_ZERO(AllocImplicitRootLeaf, ImplicitRootLeafBits); DEFINE_RECYCLER_NOTHROW_ALLOC_ZERO(AllocImplicitRoot, ImplicitRootBit); template char * AllocEnumClass(DECLSPEC_GUARD_OVERFLOW size_t size) { Assert((enumClass & EnumClassMask) != 0); //Assert((enumClass & ~EnumClassMask & ~WithBarrierBit) == 0); return AllocWithAttributes<(ObjectInfoBits)(enumClass), /* nothrow = */ false>(size); } template char * AllocWithInfoBits(DECLSPEC_GUARD_OVERFLOW size_t size) { return AllocWithAttributes(size); } template char * AllocVisitedHost(DECLSPEC_GUARD_OVERFLOW size_t size) { return AllocWithAttributes(size); } template RecyclerWeakReference* CreateWeakReferenceHandle(T* pStrongReference); #if ENABLE_WEAK_REFERENCE_REGIONS template RecyclerWeakReferenceRegionItem* CreateWeakReferenceRegion(size_t count); #endif uint GetWeakReferenceCleanupId() const { return weakReferenceCleanupId; } template bool FindOrCreateWeakReferenceHandle(T* pStrongReference, RecyclerWeakReference **ppWeakRef); template bool TryGetWeakReferenceHandle(T* pStrongReference, RecyclerWeakReference **weakReference); template char* GetAddressOfAllocator(size_t sizeCat) { Assert(HeapInfo::IsAlignedSmallObjectSize(sizeCat)); return (char*)this->GetHeapInfo()->template GetBucket<(ObjectInfoBits)(attributes & GetBlockTypeBitMask)>(sizeCat).GetAllocator(); } template uint32 GetEndAddressOffset(size_t sizeCat) { Assert(HeapInfo::IsAlignedSmallObjectSize(sizeCat)); return this->GetHeapInfo()->template GetBucket<(ObjectInfoBits)(attributes & GetBlockTypeBitMask)>(sizeCat).GetAllocator()->GetEndAddressOffset(); } template uint32 GetFreeObjectListOffset(size_t sizeCat) { Assert(HeapInfo::IsAlignedSmallObjectSize(sizeCat)); return this->GetHeapInfo()->template GetBucket<(ObjectInfoBits)(attributes & GetBlockTypeBitMask)>(sizeCat).GetAllocator()->GetFreeObjectListOffset(); } void GetNormalHeapBlockAllocatorInfoForNativeAllocation(size_t sizeCat, void*& allocatorAddress, uint32& endAddressOffset, uint32& freeListOffset, bool allowBumpAllocation, bool isOOPJIT); static void GetNormalHeapBlockAllocatorInfoForNativeAllocation(void* recyclerAddr, size_t sizeCat, void*& allocatorAddress, uint32& endAddressOffset, uint32& freeListOffset, bool allowBumpAllocation, bool isOOPJIT); bool AllowNativeCodeBumpAllocation(); static void TrackNativeAllocatedMemoryBlock(Recycler * recycler, void * memBlock, size_t sizeCat); void Free(void* buffer, size_t size) { Assert(false); } bool ExplicitFreeLeaf(void* buffer, size_t size); bool ExplicitFreeNonLeaf(void* buffer, size_t size); template bool ExplicitFreeInternalWrapper(void* buffer, size_t allocSize); template bool ExplicitFreeInternal(void* buffer, size_t size, size_t sizeCat); size_t GetAllocSize(size_t size); template void SetExplicitFreeBitOnSmallBlock(HeapBlock* heapBlock, size_t sizeCat, void* buffer, ObjectInfoBits attributes); char* HeapAllocR(HeapInfo* eHeap, DECLSPEC_GUARD_OVERFLOW size_t size) { return RealAlloc(eHeap, size); } void HeapFree(HeapInfo* eHeap,void* candidate); void EnumerateObjects(ObjectInfoBits infoBits, void (*CallBackFunction)(void * address, size_t size)); void RootAddRef(void* obj, uint *count = nullptr); void RootRelease(void* obj, uint *count = nullptr); template inline char* RealAlloc(HeapInfo* heap, DECLSPEC_GUARD_OVERFLOW size_t size); template inline char* RealAllocFromBucket(HeapInfo* heap, DECLSPEC_GUARD_OVERFLOW size_t size); void EnterIdleDecommit(); void LeaveIdleDecommit(); void DisposeObjects(); BOOL IsValidObject(void* candidate, size_t minimumSize = 0); #if DBG void SetDisableThreadAccessCheck(); void SetDisableConcurrentThreadExitedCheck(); void CheckAllocExternalMark() const; BOOL IsFreeObject(void * candidate); BOOL IsReentrantState() const; #endif #if DBG_DUMP void PrintMarkStack(); #endif #ifdef PROFILE_EXEC Js::Profiler * GetProfiler() const { return this->profiler; } ArenaAllocator * AddBackgroundProfilerArena(); void ReleaseBackgroundProfilerArena(ArenaAllocator * arena); void SetProfiler(Js::Profiler * profiler, Js::Profiler * backgroundProfiler); #endif #ifdef RECYCLER_MEMORY_VERIFY BOOL VerifyEnabled() const { return verifyEnabled; } uint GetVerifyPad() const { return verifyPad; } void Verify(Js::Phase phase); static void VerifyCheck(BOOL cond, char16 const * msg, void * address, void * corruptedAddress); static void VerifyCheckFill(void * address, size_t size); void FillCheckPad(void * address, size_t size, size_t alignedAllocSize, bool objectAlreadyInitialized); void FillCheckPad(void * address, size_t size, size_t alignedAllocSize) { FillCheckPad(address, size, alignedAllocSize, false); } static void FillPadNoCheck(void * address, size_t size, size_t alignedAllocSize, bool objectAlreadyInitialized); void VerifyCheckPad(void * address, size_t size); void VerifyCheckPadExplicitFreeList(void * address, size_t size); static const byte VerifyMemFill = 0xCA; #endif #ifdef RECYCLER_ZERO_MEM_CHECK void VerifyZeroFill(void * address, size_t size); void VerifyLargeAllocZeroFill(void * address, size_t size, ObjectInfoBits attributes); #endif #ifdef RECYCLER_DUMP_OBJECT_GRAPH bool DumpObjectGraph(RecyclerObjectGraphDumper::Param * param = nullptr); void DumpObjectDescription(void *object); #endif #ifdef LEAK_REPORT void ReportLeaks(); void ReportLeaksOnProcessDetach(); #endif #ifdef CHECK_MEMORY_LEAK void CheckLeaks(char16 const * header); void CheckLeaksOnProcessDetach(char16 const * header); #endif #ifdef RECYCLER_TRACE void SetDomCollect(bool isDomCollect) { collectionParam.domCollect = isDomCollect; } void CaptureCollectionParam(CollectionFlags flags, bool repeat = false); #endif private: // RecyclerRootPtr has implicit conversion to pointers, prevent it to be // passed to RootAddRef/RootRelease directly template void RootAddRef(RecyclerRootPtr& ptr, uint *count = nullptr); template void RootRelease(RecyclerRootPtr& ptr, uint *count = nullptr); template BOOL CollectInternal(); template BOOL Collect(); template BOOL CollectWithHeuristic(); template BOOL CollectWithExhaustiveCandidate(); template BOOL GetPartialFlag(); bool NeedExhaustiveRepeatCollect() const; #if DBG bool ExpectStackSkip() const; #endif static size_t const InvalidScanRootBytes = (size_t)-1; // Small Allocator template void AddSmallAllocator(SmallHeapBlockAllocatorType * allocator, size_t sizeCat); template void RemoveSmallAllocator(SmallHeapBlockAllocatorType * allocator, size_t sizeCat); template char * SmallAllocatorAlloc(SmallHeapBlockAllocatorType * allocator, size_t sizeCat, size_t size); // Allocation template inline char * AllocWithAttributesInlined(DECLSPEC_GUARD_OVERFLOW size_t size); template char * AllocWithAttributes(DECLSPEC_GUARD_OVERFLOW size_t size) { return AllocWithAttributesInlined(size); } template inline char* AllocZeroWithAttributesInlined(DECLSPEC_GUARD_OVERFLOW size_t size); template char* AllocZeroWithAttributes(DECLSPEC_GUARD_OVERFLOW size_t size) { return AllocZeroWithAttributesInlined(size); } char* AllocWeakReferenceEntry(DECLSPEC_GUARD_OVERFLOW size_t size) { return AllocWithAttributes(size); } bool NeedDisposeTimed() { DWORD ticks = ::GetTickCount(); return (ticks > tickCountNextDispose && this->hasDisposableObject); } char* TryLargeAlloc(HeapInfo* heap, DECLSPEC_GUARD_OVERFLOW size_t size, ObjectInfoBits attributes, bool nothrow); template char* LargeAlloc(HeapInfo* heap, DECLSPEC_GUARD_OVERFLOW size_t size, ObjectInfoBits attributes); void OutOfMemory(); // Collection BOOL DoCollect(CollectionFlags flags); BOOL DoCollectWrapped(CollectionFlags flags); BOOL CollectOnAllocatorThread(); #if DBG void ResetThreadId(); #endif template size_t ScanPinnedObjects(); size_t ScanStack(); size_t ScanArena(ArenaData * alloc, bool background); void ScanImplicitRoots(); void ScanInitialImplicitRoots(); void ScanNewImplicitRoots(); size_t FindRoots(); size_t TryMarkArenaMemoryBlockList(ArenaMemoryBlock * memoryBlocks); size_t TryMarkBigBlockList(BigBlock * memoryBlocks); #if ENABLE_CONCURRENT_GC #if FALSE // REVIEW: remove this code since not using size_t TryMarkBigBlockListWithWriteWatch(BigBlock * memoryBlocks); #endif #endif // Mark void ResetMarks(ResetMarkFlags flags); void Mark(); bool EndMark(); bool EndMarkCheckOOMRescan(); void EndMarkOnLowMemory(); #if ENABLE_CONCURRENT_GC void DoParallelMark(); void DoBackgroundParallelMark(); #endif size_t RootMark(CollectionState markState); void ProcessMark(bool background); void ProcessParallelMark(bool background, MarkContext * markContext); template void ProcessMarkContext(MarkContext * markContext); public: bool IsObjectMarked(void* candidate) { return this->heapBlockMap.IsMarked(candidate); } #ifdef RECYCLER_STRESS bool StressCollectNow(); #endif private: HeapBlock* FindHeapBlock(void * candidate); struct FindBlockCache { FindBlockCache(): heapBlock(nullptr), candidate(nullptr) { } HeapBlock* heapBlock; void* candidate; } blockCache; inline void ScanObjectInline(void ** obj, size_t byteCount); inline void ScanObjectInlineInterior(void ** obj, size_t byteCount); template inline void ScanMemoryInline(void ** obj, size_t byteCount ADDRESS_SANITIZER_APPEND(RecyclerScanMemoryType scanMemoryType = RecyclerScanMemoryType::General)); template void ScanMemory(void ** obj, size_t byteCount) { if (byteCount != 0) { ScanMemoryInline(obj, byteCount); } } bool AddMark(void * candidate, size_t byteCount) throw(); #ifdef RECYCLER_VISITED_HOST bool AddPreciselyTracedMark(IRecyclerVisitedObject * candidate) throw(); #endif // Sweep #if ENABLE_PARTIAL_GC bool Sweep(size_t rescanRootBytes = (size_t)-1, bool concurrent = false, bool adjustPartialHeuristics = false); #else bool Sweep(bool concurrent = false); #endif void SweepWeakReference(); void SweepHeap(bool concurrent, RecyclerSweepManager& recyclerSweepManager); void FinishSweep(RecyclerSweepManager& recyclerSweepManager); #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP void DoTwoPassConcurrentSweepPreCheck(); void FinishSweepPrep(); void FinishConcurrentSweepPass1(); void FinishConcurrentSweep(); #endif bool FinishDisposeObjects(); template bool FinishDisposeObjectsWrapped(); // end collection void FinishCollection(); void FinishCollection(bool needConcurrentSweep); void EndCollection(); void ResetCollectionState(); void ResetMarkCollectionState(); void ResetHeuristicCounters(); void ResetPartialHeuristicCounters(); BOOL IsMarkState() const; BOOL IsFindRootsState() const; BOOL IsInThreadFindRootsState() const; template void CollectionBegin(); template void CollectionEnd(); #if ENABLE_PARTIAL_GC void ProcessClientTrackedObjects(); bool PartialCollect(bool concurrent); void FinishPartialCollect(RecyclerSweepManager * recyclerSweep = nullptr); void ClearPartialCollect(); #if ENABLE_CONCURRENT_GC void BackgroundFinishPartialCollect(RecyclerSweepManager * recyclerSweep); #endif #endif size_t RescanMark(DWORD waitTime); size_t FinishMark(DWORD waitTime); size_t FinishMarkRescan(bool background); #if ENABLE_CONCURRENT_GC void ProcessTrackedObjects(); #endif BOOL IsAllocatableCallbackState() { return (collectionState & (Collection_PostSweepRedeferralCallback | Collection_PostCollectionCallback)); } #if ENABLE_CONCURRENT_GC // Concurrent GC BOOL IsConcurrentEnabled() const { return this->enableConcurrentMark || this->enableParallelMark || this->enableConcurrentSweep; } BOOL IsConcurrentMarkState() const; BOOL IsConcurrentMarkExecutingState() const; BOOL IsConcurrentResetMarksState() const; BOOL IsConcurrentFindRootState() const; BOOL IsConcurrentExecutingState() const; BOOL IsConcurrentSweepExecutingState() const; BOOL IsConcurrentSweepSetupState() const; BOOL IsConcurrentSweepState() const; BOOL IsConcurrentState() const; BOOL InConcurrentSweep() { return ((collectionState & Collection_ConcurrentSweep) == Collection_ConcurrentSweep); } #if ENABLE_ALLOCATIONS_DURING_CONCURRENT_SWEEP bool AllowAllocationsDuringConcurrentSweep() { return this->allowAllocationsDuringConcurrentSweepForCollection; } #endif #if DBG BOOL IsConcurrentFinishedState() const; #endif // DBG bool InitializeConcurrent(JsUtil::ThreadService* threadService); bool AbortConcurrent(bool restoreState); void FinalizeConcurrent(bool restoreState); static unsigned int CALLBACK StaticThreadProc(LPVOID lpParameter); static int ExceptFilter(LPEXCEPTION_POINTERS pEP); DWORD ThreadProc(); void DoBackgroundWork(bool forceForeground = false); static void CALLBACK StaticBackgroundWorkCallback(void * callbackData); BOOL CollectOnConcurrentThread(); bool StartConcurrent(CollectionState const state); BOOL StartBackgroundMarkCollect(); BOOL StartSynchronousBackgroundMark(); BOOL StartAsynchronousBackgroundMark(); BOOL StartBackgroundMark(bool foregroundResetMark, bool foregroundFindRoots); BOOL StartConcurrentSweepCollect(); template BOOL TryFinishConcurrentCollect(); BOOL WaitForConcurrentThread(DWORD waitTime, RecyclerWaitReason caller = RecyclerWaitReason::Other); void FlushBackgroundPages(); BOOL FinishConcurrentCollect(CollectionFlags flags); void FinishTransferSwept(CollectionFlags flags); BOOL FinishConcurrentCollectWrapped(CollectionFlags flags); void BackgroundMark(); void BackgroundMarkWeakRefs(); void BackgroundResetMarks(); void PrepareBackgroundFindRoots(); void RevertPrepareBackgroundFindRoots(); size_t BackgroundFindRoots(); size_t BackgroundScanStack(); size_t BackgroundRepeatMark(); size_t BackgroundRescan(RescanFlags rescanFlags); void BackgroundResetWriteWatchAll(); size_t BackgroundFinishMark(); char* GetScriptThreadStackTop(); void SweepPendingObjects(RecyclerSweepManager& recyclerSweepManager); void ConcurrentTransferSweptObjects(RecyclerSweepManager& recyclerSweepManager); #if ENABLE_PARTIAL_GC void ConcurrentPartialTransferSweptObjects(RecyclerSweepManager& recyclerSweepManager); #endif // ENABLE_PARTIAL_GC #endif // ENABLE_CONCURRENT_GC bool ForceSweepObject(); void NotifyFree(__in char * address, size_t size); template void NotifyFree(T * heapBlock); void CleanupPendingUnroot(); #ifdef ENABLE_JS_ETW ULONG EventWriteFreeMemoryBlock(HeapBlock* heapBlock); void FlushFreeRecord(); void AppendFreeMemoryETWRecord(__in char *address, size_t size); static const uint BulkFreeMemoryCount = 400; uint bulkFreeMemoryWrittenCount; struct ETWFreeRecord { char* memoryAddress; uint32 objectSize; }; ETWFreeRecord etwFreeRecords[BulkFreeMemoryCount]; #endif template bool IntegrateBlock(char * blockAddress, PageSegment * segment, size_t allocSize, size_t objectSize); template friend class SmallHeapBlockT; template friend class SmallNormalHeapBlockT; template friend class SmallLeafHeapBlockT; template friend class SmallFinalizableHeapBlockT; #ifdef RECYCLER_VISITED_HOST template friend class SmallRecyclerVisitedHostHeapBlockT; #endif friend class LargeHeapBlock; friend class HeapInfo; friend class HeapInfoManager; friend class LargeHeapBucket; friend class ThreadContext; template friend class HeapBucketT; template friend class SmallNormalHeapBucketBase; template friend class RecyclerFastAllocator; #ifdef RECYCLER_TRACE void PrintCollectTrace(Js::Phase phase, bool finish = false, bool noConcurrentWork = false); #endif #ifdef RECYCLER_VERIFY_MARK void VerifyMark(); void VerifyMarkRoots(); void VerifyMarkStack(); void VerifyMarkArena(ArenaData * arena); void VerifyMarkBigBlockList(BigBlock * memoryBlocks); void VerifyMarkArenaMemoryBlockList(ArenaMemoryBlock * memoryBlocks); bool VerifyMark(void * objectAddress, void * target); bool VerifyMark(void * target); #endif #if DBG_DUMP bool forceTraceMark; #endif bool isHeapEnumInProgress; #if DBG bool allowAllocationDuringHeapEnum; bool allowAllocationDuringRenentrance; #ifdef ENABLE_PROJECTION bool isInRefCountTrackingForProjection; #endif #endif // There are two scenarios we allow limited allocation but disallow GC during those allocations: // in heapenum when we allocate PropertyRecord, and // in projection ExternalMark allowing allocating VarToDispEx. This is the common flag // while we have debug only flag for each of the two scenarios. bool isCollectionDisabled; #ifdef ENABLE_BASIC_TELEMETRY RecyclerTelemetryInfo& GetRecyclerTelemetryInfo() { return this->telemetryStats; } #endif #ifdef TRACK_ALLOC public: Recycler * TrackAllocInfo(TrackAllocData const& data); void ClearTrackAllocInfo(TrackAllocData* data = NULL); #ifdef PROFILE_RECYCLER_ALLOC void PrintAllocStats(); private: static bool DoProfileAllocTracker(); void InitializeProfileAllocTracker(); void TrackUnallocated(__in char* address, __in char *endAddress, size_t sizeCat); void TrackAllocCore(void * object, size_t size, const TrackAllocData& trackAllocData, bool traceLifetime = false); void* TrackAlloc(void * object, size_t size, const TrackAllocData& trackAllocData, bool traceLifetime = false); void TrackIntegrate(__in_ecount(blockSize) char * blockAddress, size_t blockSize, size_t allocSize, size_t objectSize, const TrackAllocData& trackAllocData); BOOL TrackFree(const char* address, size_t size); void TrackAllocWeakRef(RecyclerWeakReferenceBase * weakRef); void TrackFreeWeakRef(RecyclerWeakReferenceBase * weakRef); struct TrackerData { TrackerData(type_info const * typeinfo, bool isArray) : typeinfo(typeinfo), isArray(isArray), ItemSize(0), ItemCount(0), AllocCount(0), ReqSize(0), AllocSize(0), FreeCount(0), FreeSize(0), TraceLifetime(false) #ifdef PERF_COUNTERS , counter(PerfCounter::RecyclerTrackerCounterSet::GetPerfCounter(typeinfo, isArray)) , sizeCounter(PerfCounter::RecyclerTrackerCounterSet::GetPerfSizeCounter(typeinfo, isArray)) #endif { } type_info const * typeinfo; bool isArray; #ifdef TRACE_OBJECT_LIFETIME bool TraceLifetime; #endif size_t ItemSize; size_t ItemCount; int AllocCount; int64 ReqSize; int64 AllocSize; int FreeCount; int64 FreeSize; #ifdef PERF_COUNTERS PerfCounter::Counter& counter; PerfCounter::Counter& sizeCounter; #endif static TrackerData EmptyData; static TrackerData ExplicitFreeListObjectData; }; TrackerData * GetTrackerData(void * address); void SetTrackerData(void * address, TrackerData * data); struct TrackerItem { TrackerItem(type_info const * typeinfo) : instanceData(typeinfo, false), arrayData(typeinfo, true) #ifdef PERF_COUNTERS , weakRefCounter(PerfCounter::RecyclerTrackerCounterSet::GetWeakRefPerfCounter(typeinfo)) #endif {} TrackerData instanceData; TrackerData arrayData; #ifdef PERF_COUNTERS PerfCounter::Counter& weakRefCounter; #endif }; typedef JsUtil::BaseDictionary TypeInfotoTrackerItemMap; typedef JsUtil::BaseDictionary PointerToTrackerDataMap; TypeInfotoTrackerItemMap * trackerDictionary; CriticalSection * trackerCriticalSection; #endif TrackAllocData nextAllocData; #endif public: // Enumeration class AutoSetupRecyclerForNonCollectingMark { private: Recycler& m_recycler; bool m_setupDone; CollectionState m_previousCollectionState; #ifdef RECYCLER_STATS RecyclerCollectionStats m_previousCollectionStats; #endif public: AutoSetupRecyclerForNonCollectingMark(Recycler& recycler, bool setupForHeapEnumeration = false); ~AutoSetupRecyclerForNonCollectingMark(); void DoCommonSetup(); void SetupForHeapEnumeration(); }; friend class RecyclerHeapObjectInfo; bool FindImplicitRootObject(void* candidate, RecyclerHeapObjectInfo& heapObject); bool FindHeapObject(void* candidate, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject); bool FindHeapObjectWithClearedAllocators(void* candidate, RecyclerHeapObjectInfo& heapObject); bool IsCollectionDisabled() const { return isCollectionDisabled; } bool IsHeapEnumInProgress() const { Assert(isHeapEnumInProgress ? isCollectionDisabled : true); return isHeapEnumInProgress; } #if DBG // There are limited cases that we have to allow allocation during heap enumeration. GC is explicitly // disabled during heap enumeration for these limited cases. (See DefaultRecyclerCollectionWrapper) // The only case of allocation right now is allocating property record for string based type handler // so we can use the propertyId as the relation Id. // Allocation during enumeration is still frown upon and should still be avoid if possible. bool AllowAllocationDuringHeapEnum() const { return allowAllocationDuringHeapEnum; } class AutoAllowAllocationDuringHeapEnum : public AutoBooleanToggle { public: AutoAllowAllocationDuringHeapEnum(Recycler * recycler) : AutoBooleanToggle(&recycler->allowAllocationDuringHeapEnum) {}; }; #ifdef ENABLE_PROJECTION bool IsInRefCountTrackingForProjection() const { return isInRefCountTrackingForProjection;} class AutoIsInRefCountTrackingForProjection : public AutoBooleanToggle { public: AutoIsInRefCountTrackingForProjection(Recycler * recycler) : AutoBooleanToggle(&recycler->isInRefCountTrackingForProjection) {}; }; #endif #endif class AutoAllowAllocationDuringReentrance : public AutoBooleanToggle { public: AutoAllowAllocationDuringReentrance(Recycler * recycler) : AutoBooleanToggle(&recycler->isCollectionDisabled) #if DBG , allowAllocationDuringRenentrance(&recycler->allowAllocationDuringRenentrance) #endif {}; #if DBG private: AutoBooleanToggle allowAllocationDuringRenentrance; #endif }; #ifdef HEAP_ENUMERATION_VALIDATION typedef void(*PostHeapEnumScanCallback)(const HeapObject& heapObject, void *data); PostHeapEnumScanCallback pfPostHeapEnumScanCallback; void *postHeapEnunScanData; void PostHeapEnumScan(PostHeapEnumScanCallback callback, void*data); bool IsPostEnumHeapValidationInProgress() const { return pfPostHeapEnumScanCallback != NULL; } #endif public: void* GetRealAddressFromInterior(void* candidate); private: void BeginNonCollectingMark(); void EndNonCollectingMark(); #if defined(RECYCLER_DUMP_OBJECT_GRAPH) || defined(LEAK_REPORT) || defined(CHECK_MEMORY_LEAK) public: bool IsInDllCanUnloadNow() const { return inDllCanUnloadNow; } bool IsInDetachProcess() const { return inDetachProcess; } void SetInDllCanUnloadNow(); void SetInDetachProcess(); private: bool inDllCanUnloadNow; bool inDetachProcess; bool isPrimaryMarkContextInitialized; #endif #if defined(LEAK_REPORT) || defined(CHECK_MEMORY_LEAK) template void ReportOnProcessDetach(Fn fn); void PrintPinnedObjectStackTraces(); #endif public: typedef void (CALLBACK *ObjectBeforeCollectCallback)(void* object, void* callbackState); // same as jsrt JsObjectBeforeCollectCallback // same as jsrt JsObjectBeforeCollectCallbackWrapper typedef void (CALLBACK *ObjectBeforeCollectCallbackWrapper)(ObjectBeforeCollectCallback callback, void* object, void* callbackState, void* threadContext); void SetObjectBeforeCollectCallback(void* object, ObjectBeforeCollectCallback callback, void* callbackState, ObjectBeforeCollectCallbackWrapper callbackWrapper, void* threadContext); void ClearObjectBeforeCollectCallbacks(); bool IsInObjectBeforeCollectCallback() const { return objectBeforeCollectCallbackState != ObjectBeforeCollectCallback_None; } private: struct ObjectBeforeCollectCallbackData { ObjectBeforeCollectCallback callback; void* callbackState; void* threadContext; ObjectBeforeCollectCallbackWrapper callbackWrapper; ObjectBeforeCollectCallbackData() {} ObjectBeforeCollectCallbackData(ObjectBeforeCollectCallbackWrapper callbackWrapper, ObjectBeforeCollectCallback callback, void* callbackState, void* threadContext) : callbackWrapper(callbackWrapper), callback(callback), callbackState(callbackState), threadContext(threadContext) {} }; typedef JsUtil::BaseDictionary ObjectBeforeCollectCallbackMap; ObjectBeforeCollectCallbackMap* objectBeforeCollectCallbackMap; enum ObjectBeforeCollectCallbackState { ObjectBeforeCollectCallback_None, ObjectBeforeCollectCallback_Normal, // Normal GC BeforeCollect callback ObjectBeforeCollectCallback_Shutdown, // At shutdown invoke all BeforeCollect callback } objectBeforeCollectCallbackState; bool ProcessObjectBeforeCollectCallbacks(bool atShutdown = false); #if GLOBAL_ENABLE_WRITE_BARRIER private: typedef JsUtil::BaseDictionary PendingWriteBarrierBlockMap; PendingWriteBarrierBlockMap pendingWriteBarrierBlockMap; public: void RegisterPendingWriteBarrierBlock(void* address, size_t bytes); void UnRegisterPendingWriteBarrierBlock(void* address); #endif #if DBG && GLOBAL_ENABLE_WRITE_BARRIER private: static Recycler* recyclerList; static CriticalSection recyclerListLock; Recycler* next; public: static void WBSetBitJIT(char* addr) { return WBSetBit(addr); } static void WBSetBit(char* addr); static void WBSetBitRange(char* addr, uint length); static void WBVerifyBitIsSet(char* addr, char* target); static bool WBCheckIsRecyclerAddress(char* addr); #endif #ifdef RECYCLER_FINALIZE_CHECK void VerifyFinalize(); #endif }; class RecyclerHeapObjectInfo { void* m_address; Recycler * m_recycler; HeapBlock* m_heapBlock; #if LARGEHEAPBLOCK_ENCODING union { byte * m_attributes; LargeObjectHeader * m_largeHeapBlockHeader; }; bool isUsingLargeHeapBlock = false; #else byte * m_attributes; #endif public: RecyclerHeapObjectInfo() : m_address(NULL), m_recycler(NULL), m_heapBlock(NULL), m_attributes(NULL) {} RecyclerHeapObjectInfo(void* address, Recycler * recycler, HeapBlock* heapBlock, byte * attributes) : m_address(address), m_recycler(recycler), m_heapBlock(heapBlock), m_attributes(attributes) { } void* GetObjectAddress() const { return m_address; } #ifdef RECYCLER_PAGE_HEAP bool IsPageHeapAlloc() const { return isUsingLargeHeapBlock && ((LargeHeapBlock*)m_heapBlock)->InPageHeapMode(); } void PageHeapLockPages() const { Assert(IsPageHeapAlloc()); ((LargeHeapBlock*)m_heapBlock)->PageHeapLockPages(); } #endif bool IsLeaf() const { #if LARGEHEAPBLOCK_ENCODING if (isUsingLargeHeapBlock) { return (m_largeHeapBlockHeader->GetAttributes(m_recycler->Cookie) & LeafBit) != 0; } #endif return ((*m_attributes & LeafBit) != 0 || this->m_heapBlock->IsLeafBlock()); } bool IsImplicitRoot() const { #if LARGEHEAPBLOCK_ENCODING if (isUsingLargeHeapBlock) { return (m_largeHeapBlockHeader->GetAttributes(m_recycler->Cookie) & ImplicitRootBit) != 0; } #endif return (*m_attributes & ImplicitRootBit) != 0; } bool IsObjectMarked() const { Assert(m_recycler); return m_recycler->heapBlockMap.IsMarked(m_address); } void SetObjectMarked() { Assert(m_recycler); m_recycler->heapBlockMap.SetMark(m_address); } ObjectInfoBits GetAttributes() const { #if LARGEHEAPBLOCK_ENCODING if (isUsingLargeHeapBlock) { return (ObjectInfoBits)m_largeHeapBlockHeader->GetAttributes(m_recycler->Cookie); } #endif return (ObjectInfoBits)*m_attributes; } size_t GetSize() const; #if LARGEHEAPBLOCK_ENCODING void SetLargeHeapBlockHeader(LargeObjectHeader * largeHeapBlockHeader) { m_largeHeapBlockHeader = largeHeapBlockHeader; isUsingLargeHeapBlock = true; } #endif bool SetMemoryProfilerHasEnumerated() { Assert(m_heapBlock); #if LARGEHEAPBLOCK_ENCODING if (isUsingLargeHeapBlock) { return SetMemoryProfilerHasEnumeratedForLargeHeapBlock(); } #endif bool wasMemoryProfilerOldObject = (*m_attributes & MemoryProfilerOldObjectBit) != 0; *m_attributes |= MemoryProfilerOldObjectBit; return wasMemoryProfilerOldObject; } bool ClearImplicitRootBit() { // This can only be called on the main thread for non-finalizable block // As finalizable block requires that the bit not be change during concurrent mark // since the background thread change the NewTrackBit Assert(!m_heapBlock->IsAnyFinalizableBlock()); #ifdef RECYCLER_PAGE_HEAP Recycler* recycler = this->m_recycler; if (recycler->IsPageHeapEnabled() && recycler->ShouldCapturePageHeapFreeStack()) { #ifdef STACK_BACK_TRACE if (this->isUsingLargeHeapBlock) { LargeHeapBlock* largeHeapBlock = (LargeHeapBlock*)this->m_heapBlock; if (largeHeapBlock->InPageHeapMode()) { largeHeapBlock->CapturePageHeapFreeStack(); } } #endif } #endif #if LARGEHEAPBLOCK_ENCODING if (isUsingLargeHeapBlock) { return ClearImplicitRootBitsForLargeHeapBlock(); } #endif Assert(m_attributes); bool wasImplicitRoot = (*m_attributes & ImplicitRootBit) != 0; *m_attributes &= ~ImplicitRootBit; return wasImplicitRoot; } void ExplicitFree() { if (*m_attributes == ObjectInfoBits::LeafBit) { m_recycler->ExplicitFreeLeaf(m_address, GetSize()); } else { Assert(*m_attributes == ObjectInfoBits::NoBit); m_recycler->ExplicitFreeNonLeaf(m_address, GetSize()); } } #if LARGEHEAPBLOCK_ENCODING bool ClearImplicitRootBitsForLargeHeapBlock() { Assert(m_largeHeapBlockHeader); byte attributes = m_largeHeapBlockHeader->GetAttributes(m_recycler->Cookie); bool wasImplicitRoot = (attributes & ImplicitRootBit) != 0; m_largeHeapBlockHeader->SetAttributes(m_recycler->Cookie, attributes & ~ImplicitRootBit); return wasImplicitRoot; } bool SetMemoryProfilerHasEnumeratedForLargeHeapBlock() { Assert(m_largeHeapBlockHeader); byte attributes = m_largeHeapBlockHeader->GetAttributes(m_recycler->Cookie); bool wasMemoryProfilerOldObject = (attributes & MemoryProfilerOldObjectBit) != 0; m_largeHeapBlockHeader->SetAttributes(m_recycler->Cookie, attributes | MemoryProfilerOldObjectBit); return wasMemoryProfilerOldObject; } #endif }; // A fake heap block to replace the original heap block where the strong ref is when it has been collected // as the original heap block may have been freed class CollectedRecyclerWeakRefHeapBlock : public HeapBlock { public: #if DBG && GLOBAL_ENABLE_WRITE_BARRIER virtual void WBVerifyBitIsSet(char* addr) override { Assert(false); } virtual void WBSetBit(char* addr) override { Assert(false); } virtual void WBSetBitRange(char* addr, uint count) override { Assert(false); } virtual void WBClearBit(char* addr) override { Assert(false); } virtual void WBClearObject(char* addr) override { Assert(false); } #endif #if DBG virtual HeapInfo * GetHeapInfo() const override { Assert(false); return nullptr; } virtual BOOL IsFreeObject(void* objectAddress) override { Assert(false); return false; } #endif virtual BOOL IsValidObject(void* objectAddress) override { Assert(false); return false; } virtual byte* GetRealAddressFromInterior(void* interiorAddress) override { Assert(false); return nullptr; } virtual size_t GetObjectSize(void* object) const override { Assert(false); return 0; } virtual bool FindHeapObject(void* objectAddress, Recycler * recycler, FindHeapObjectFlags flags, RecyclerHeapObjectInfo& heapObject) override { Assert(false); return false; } virtual bool TestObjectMarkedBit(void* objectAddress) override { Assert(false); return false; } virtual void SetObjectMarkedBit(void* objectAddress) override { Assert(false); } #ifdef RECYCLER_VERIFY_MARK virtual bool VerifyMark(void * objectAddress, void * target) override { Assert(false); return false; } #endif #ifdef RECYCLER_PERF_COUNTERS virtual void UpdatePerfCountersOnFree() override { Assert(false); } #endif #ifdef PROFILE_RECYCLER_ALLOC virtual void * GetTrackerData(void * address) override { Assert(false); return nullptr; } virtual void SetTrackerData(void * address, void * data) override { Assert(false); } #endif static CollectedRecyclerWeakRefHeapBlock Instance; private: CollectedRecyclerWeakRefHeapBlock() : HeapBlock(BlockTypeCount) { #if ENABLE_CONCURRENT_GC isPendingConcurrentSweep = false; #endif } }; class AutoIdleDecommit { public: AutoIdleDecommit(Recycler * recycler) : recycler(recycler) { recycler->EnterIdleDecommit(); } ~AutoIdleDecommit() { recycler->LeaveIdleDecommit(); } private: Recycler * recycler; }; template void Recycler::AddSmallAllocator(SmallHeapBlockAllocatorType * allocator, size_t sizeCat) { this->GetDefaultHeapInfo()->AddSmallAllocator(allocator, sizeCat); } template void Recycler::RemoveSmallAllocator(SmallHeapBlockAllocatorType * allocator, size_t sizeCat) { this->GetDefaultHeapInfo()->RemoveSmallAllocator(allocator, sizeCat); } template char * Recycler::SmallAllocatorAlloc(SmallHeapBlockAllocatorType * allocator, DECLSPEC_GUARD_OVERFLOW size_t sizeCat, size_t size) { return this->GetDefaultHeapInfo()->SmallAllocatorAlloc(this, allocator, sizeCat, size); } // Dummy recycler allocator policy classes to choose the allocation function class _RecyclerLeafPolicy; class _RecyclerNonLeafPolicy; #ifdef RECYCLER_WRITE_BARRIER class _RecyclerWriteBarrierPolicy; #endif template class _RecyclerAllocatorFunc {}; template <> class _RecyclerAllocatorFunc<_RecyclerLeafPolicy> { public: typedef char * (Recycler::*AllocFuncType)(size_t); typedef bool (Recycler::*FreeFuncType)(void*, size_t); static AllocFuncType GetAllocFunc() { return &Recycler::AllocLeaf; } static AllocFuncType GetAllocZeroFunc() { return &Recycler::AllocLeafZero; } static FreeFuncType GetFreeFunc() { return &Recycler::ExplicitFreeLeaf; } }; template <> class _RecyclerAllocatorFunc<_RecyclerNonLeafPolicy> { public: typedef char * (Recycler::*AllocFuncType)(size_t); typedef bool (Recycler::*FreeFuncType)(void*, size_t); static AllocFuncType GetAllocFunc() { return &Recycler::Alloc; } static AllocFuncType GetAllocZeroFunc() { return &Recycler::AllocZero; } static FreeFuncType GetFreeFunc() { return &Recycler::ExplicitFreeNonLeaf; } }; #ifdef RECYCLER_WRITE_BARRIER template <> class _RecyclerAllocatorFunc<_RecyclerWriteBarrierPolicy> { public: typedef char * (Recycler::*AllocFuncType)(size_t); typedef bool (Recycler::*FreeFuncType)(void*, size_t); static AllocFuncType GetAllocFunc() { return &Recycler::AllocWithBarrier; } static AllocFuncType GetAllocZeroFunc() { return &Recycler::AllocZeroWithBarrier; } static FreeFuncType GetFreeFunc() { return &Recycler::ExplicitFreeNonLeaf; } }; #endif // This is used by the compiler; when T is NOT a pointer i.e. a value type - it causes leaf allocation template class TypeAllocatorFunc : public _RecyclerAllocatorFunc<_RecyclerLeafPolicy> { }; #if GLOBAL_ENABLE_WRITE_BARRIER template class TypeAllocatorFunc : public _RecyclerAllocatorFunc<_RecyclerWriteBarrierPolicy> { }; #else // Partial template specialization; applies to T when it is a pointer template class TypeAllocatorFunc : public _RecyclerAllocatorFunc<_RecyclerNonLeafPolicy> { }; #endif // Dummy class to choose the allocation function class RecyclerLeafAllocator { public: static const bool FakeZeroLengthArray = true; }; class RecyclerNonLeafAllocator { public: static const bool FakeZeroLengthArray = true; }; class RecyclerWriteBarrierAllocator { public: static const bool FakeZeroLengthArray = true; }; // Choose RecyclerLeafAllocator / RecyclerNonLeafAllocator based on "bool isLeaf" template struct _RecyclerLeaf { typedef RecyclerLeafAllocator AllocatorType; }; template <> struct _RecyclerLeaf { typedef RecyclerNonLeafAllocator AllocatorType; }; template class ListTypeAllocatorFunc { public: // RecyclerLeafAllocator / RecyclerNonLeafAllocator based on "bool isLeaf" // used by write barrier type traits typedef typename _RecyclerLeaf::AllocatorType EffectiveAllocatorType; typedef char * (Recycler::*AllocFuncType)(size_t); typedef bool (Recycler::*FreeFuncType)(void*, size_t); static AllocFuncType GetAllocFunc() { return isLeaf ? &Recycler::AllocLeaf : &Recycler::Alloc; } static FreeFuncType GetFreeFunc() { if (isLeaf) { return &Recycler::ExplicitFreeLeaf; } else { return &Recycler::ExplicitFreeNonLeaf; } } }; // Partial template specialization to allocate as non leaf template class TypeAllocatorFunc : #if GLOBAL_ENABLE_WRITE_BARRIER public _RecyclerAllocatorFunc<_RecyclerWriteBarrierPolicy> #else public _RecyclerAllocatorFunc<_RecyclerNonLeafPolicy> #endif { }; #ifdef RECYCLER_WRITE_BARRIER template class TypeAllocatorFunc : public _RecyclerAllocatorFunc<_RecyclerWriteBarrierPolicy> { }; #endif template class TypeAllocatorFunc : public _RecyclerAllocatorFunc<_RecyclerLeafPolicy> { }; template struct AllocatorInfo { typedef Recycler AllocatorType; typedef TypeAllocatorFunc AllocatorFunc; typedef _RecyclerAllocatorFunc<_RecyclerNonLeafPolicy> InstAllocatorFunc; // By default any instance considered non-leaf }; template struct AllocatorInfo { typedef Recycler AllocatorType; typedef TypeAllocatorFunc AllocatorFunc; typedef TypeAllocatorFunc InstAllocatorFunc; // Same as TypeAllocatorFunc }; template struct AllocatorInfo { typedef Recycler AllocatorType; typedef TypeAllocatorFunc AllocatorFunc; typedef TypeAllocatorFunc InstAllocatorFunc; // Same as TypeAllocatorFunc }; template struct AllocatorInfo { typedef Recycler AllocatorType; typedef TypeAllocatorFunc AllocatorFunc; typedef TypeAllocatorFunc InstAllocatorFunc; // Same as TypeAllocatorFunc }; template <> struct ForceNonLeafAllocator { typedef RecyclerNonLeafAllocator AllocatorType; }; template <> struct ForceNonLeafAllocator { typedef RecyclerNonLeafAllocator AllocatorType; }; template <> struct ForceLeafAllocator { typedef RecyclerLeafAllocator AllocatorType; }; template <> struct ForceLeafAllocator { typedef RecyclerLeafAllocator AllocatorType; }; // TODO: enable -profile for GC phases. // access the same profiler object from multiple GC threads which shares one recycler object, // but profiler object is not thread safe #if defined(PROFILE_EXEC) && 0 #define RECYCLER_PROFILE_EXEC_BEGIN(recycler, phase) if (recycler->profiler != nullptr) { recycler->profiler->Begin(phase); } #define RECYCLER_PROFILE_EXEC_END(recycler, phase) if (recycler->profiler != nullptr) { recycler->profiler->End(phase); } #define RECYCLER_PROFILE_EXEC_BEGIN2(recycler, phase1, phase2) if (recycler->profiler != nullptr) { recycler->profiler->Begin(phase1); recycler->profiler->Begin(phase2);} #define RECYCLER_PROFILE_EXEC_END2(recycler, phase1, phase2) if (recycler->profiler != nullptr) { recycler->profiler->End(phase1); recycler->profiler->End(phase2);} #define RECYCLER_PROFILE_EXEC_CHANGE(recycler, phase1, phase2) if (recycler->profiler != nullptr) { recycler->profiler->End(phase1); recycler->profiler->Begin(phase2); } #define RECYCLER_PROFILE_EXEC_BACKGROUND_BEGIN(recycler, phase) if (recycler->backgroundProfiler != nullptr) { recycler->backgroundProfiler->Begin(phase); } #define RECYCLER_PROFILE_EXEC_BACKGROUND_END(recycler, phase) if (recycler->backgroundProfiler != nullptr) { recycler->backgroundProfiler->End(phase); } #define RECYCLER_PROFILE_EXEC_THREAD_BEGIN(background, recycler, phase) if (background) { RECYCLER_PROFILE_EXEC_BACKGROUND_BEGIN(recycler, phase); } else { RECYCLER_PROFILE_EXEC_BEGIN(recycler, phase); } #define RECYCLER_PROFILE_EXEC_THREAD_END(background, recycler, phase) if (background) { RECYCLER_PROFILE_EXEC_BACKGROUND_END(recycler, phase); } else { RECYCLER_PROFILE_EXEC_END(recycler, phase); } #else #define RECYCLER_PROFILE_EXEC_BEGIN(recycler, phase) #define RECYCLER_PROFILE_EXEC_END(recycler, phase) #define RECYCLER_PROFILE_EXEC_BEGIN2(recycler, phase1, phase2) #define RECYCLER_PROFILE_EXEC_END2(recycler, phase1, phase2) #define RECYCLER_PROFILE_EXEC_CHANGE(recycler, phase1, phase2) #define RECYCLER_PROFILE_EXEC_BACKGROUND_BEGIN(recycler, phase) #define RECYCLER_PROFILE_EXEC_BACKGROUND_END(recycler, phase) #define RECYCLER_PROFILE_EXEC_THREAD_BEGIN(background, recycler, phase) #define RECYCLER_PROFILE_EXEC_THREAD_END(background, recycler, phase) #endif } _Ret_notnull_ inline void * __cdecl operator new(DECLSPEC_GUARD_OVERFLOW size_t byteSize, Recycler * alloc, HeapInfo * heapInfo) { return alloc->HeapAllocR(heapInfo, byteSize); } inline void __cdecl operator delete(void * obj, Recycler * alloc, HeapInfo * heapInfo) { alloc->HeapFree(heapInfo, obj); } template _Ret_notnull_ inline void * __cdecl operator new(DECLSPEC_GUARD_OVERFLOW size_t byteSize, Recycler * recycler, const InfoBitsWrapper&) { AssertCanHandleOutOfMemory(); Assert(byteSize != 0); void * buffer; if (infoBits & EnumClass_1_Bit) { buffer = recycler->AllocEnumClass(byteSize); } else { buffer = recycler->AllocWithInfoBits(byteSize); } // All of our allocation should throw on out of memory Assume(buffer != nullptr); return buffer; } #if DBG && defined(RECYCLER_VERIFY_MARK) extern bool IsLikelyRuntimeFalseReference( char* objectStartAddress, size_t offset, const char* typeName); #define DECLARE_RECYCLER_VERIFY_MARK_FRIEND() \ private: \ friend bool ::IsLikelyRuntimeFalseReference( \ char* objectStartAddress, size_t offset, const char* typeName); #define IMPLEMENT_STUB_IsLikelyRuntimeFalseReference() \ bool IsLikelyRuntimeFalseReference( \ char* objectStartAddress, size_t offset, const char* typeName) \ { return false; } #else #define DECLARE_RECYCLER_VERIFY_MARK_FRIEND() #define IMPLEMENT_STUB_IsLikelyRuntimeFalseReference() #endif template bool Recycler::DoExternalAllocation(size_t size, ExternalAllocFunc externalAllocFunc) { // Request external memory allocation if (!RequestExternalMemoryAllocation(size)) { // Attempt to free some memory then try again CollectNow(); if (!RequestExternalMemoryAllocation(size)) { return false; } } struct AutoExternalAllocation { bool allocationSucceeded = false; Recycler* recycler; size_t size; AutoExternalAllocation(Recycler* recycler, size_t size): recycler(recycler), size(size) {} // In case the externalAllocFunc throws or fails, the destructor will report the failure ~AutoExternalAllocation() { if (!allocationSucceeded) recycler->ReportExternalMemoryFailure(size); } }; AutoExternalAllocation externalAllocation(this, size); if (externalAllocFunc()) { externalAllocation.allocationSucceeded = true; return true; } return false; }