[1.8>master] [MERGE #4448 @leirocks] Delete function table in background thread

Merge pull request #4448 from leirocks:delft

RtlDeleteGrowableFunctionTable is slow. when cleaning up large amount of function
entry points in UI thread it can causes a busy hang. Moving this call to background
together with freeing jit code allocation.

Build/Chakra.Core.sln

@@ -11,6 +11,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ChakraCore", "..\bin\Chakra
 		{2F6A1847-BFAF-4B8A-9463-AC39FB46B96A} = {2F6A1847-BFAF-4B8A-9463-AC39FB46B96A}
 		{6979EC58-7A28-465C-A694-F3323A1F5401} = {6979EC58-7A28-465C-A694-F3323A1F5401}
 		{F6FAD160-5A4B-476A-93AC-33E0B3A18C0C} = {F6FAD160-5A4B-476A-93AC-33E0B3A18C0C}
+		{F48B3491-81DF-4F49-B35F-3308CBE6A379} = {F48B3491-81DF-4F49-B35F-3308CBE6A379}
 		{18CF279F-188D-4655-B03D-74F65388E7D1} = {18CF279F-188D-4655-B03D-74F65388E7D1}
 		{ABC904AD-9415-46F8-AA23-E33193F81F7C} = {ABC904AD-9415-46F8-AA23-E33193F81F7C}
 		{8C61E4E7-F0D6-420D-A352-3E6E50D406DD} = {8C61E4E7-F0D6-420D-A352-3E6E50D406DD}
@@ -755,4 +756,7 @@ Global
 		{02D4FD92-AD34-40CA-85DF-4D6C7E3A1F22} = {546172B2-F084-4363-BE35-06010663D319}
 		{F48B3491-81DF-4F49-B35F-3308CBE6A379} = {D8216B93-BD6E-4293-8D98-79CEF7CF66BC}
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {1F6CA1BC-6C01-4C82-8505-6A7690EBD556}
+	EndGlobalSection
 EndGlobal

lib/Backend/CodeGenWorkItem.cpp

@@ -205,7 +205,7 @@ void CodeGenWorkItem::OnWorkItemProcessFail(NativeCodeGenerator* codeGen)
 #if DBG
         this->allocation->allocation->isNotExecutableBecauseOOM = true;
 #endif
-        codeGen->FreeNativeCodeGenAllocation(this->allocation->allocation->address);
+        codeGen->FreeNativeCodeGenAllocation(this->allocation->allocation->address, nullptr);
     }
 }
 

lib/Backend/EmitBuffer.cpp

@@ -191,7 +191,7 @@ EmitBufferManager<TAlloc, TPreReservedAlloc, SyncObject>::NewAllocation(size_t b
 
 template <typename TAlloc, typename TPreReservedAlloc, class SyncObject>
 bool
-EmitBufferManager<TAlloc, TPreReservedAlloc, SyncObject>::FreeAllocation(void* address)
+EmitBufferManager<TAlloc, TPreReservedAlloc, SyncObject>::FreeAllocation(void* address, void** functionTable)
 {
     AutoRealOrFakeCriticalSection<SyncObject> autoCs(&this->criticalSection);
 #if _M_ARM
@@ -239,6 +239,14 @@ EmitBufferManager<TAlloc, TPreReservedAlloc, SyncObject>::FreeAllocation(void* a
             }
             VerboseHeapTrace(_u("Freeing 0x%p, allocation: 0x%p\n"), address, allocation->allocation->address);
 
+#if PDATA_ENABLED && defined(_WIN32)
+            if (functionTable && *functionTable)
+            {
+                NtdllLibrary::Instance->DeleteGrowableFunctionTable(*functionTable);
+                *functionTable = nullptr;
+            }
+#endif
+
             this->allocationHeap.Free(allocation->allocation);
             this->allocator->Free(allocation, sizeof(TEmitBufferAllocation));
 

lib/Backend/EmitBuffer.h

@@ -46,7 +46,7 @@ public:
     bool ProtectBufferWithExecuteReadWriteForInterpreter(TEmitBufferAllocation* allocation);
     bool CommitBufferForInterpreter(TEmitBufferAllocation* allocation, _In_reads_bytes_(bufferSize) BYTE* pBuffer, _In_ size_t bufferSize);
     void CompletePreviousAllocation(TEmitBufferAllocation* allocation);
-    bool FreeAllocation(void* address);
+    bool FreeAllocation(void* address, void** functionTable);
     //Ends here
 
     bool IsInHeap(void* address);

lib/Backend/InterpreterThunkEmitter.cpp

@@ -323,6 +323,10 @@ void* InterpreterThunkEmitter::ConvertToEntryPoint(PVOID dynamicInterpreterThunk
 
 bool InterpreterThunkEmitter::NewThunkBlock()
 {
+    // flush the function tables before allocating any new  code
+    // to prevent old function table is referring to new code address
+    DelayDeletingFunctionTable::Clear();
+
 #ifdef ENABLE_OOP_NATIVE_CODEGEN
     if (CONFIG_FLAG(ForceStaticInterpreterThunk))
     {
@@ -392,6 +396,10 @@ bool InterpreterThunkEmitter::NewThunkBlock()
 #ifdef ENABLE_OOP_NATIVE_CODEGEN
 bool InterpreterThunkEmitter::NewOOPJITThunkBlock()
 {
+    // flush the function tables before allocating any new  code
+    // to prevent old function table is referring to new code address
+    DelayDeletingFunctionTable::Clear();
+
     PSCRIPTCONTEXT_HANDLE remoteScriptContext = this->scriptContext->GetRemoteScriptAddr();
     if (!JITManager::GetJITManager()->IsConnected())
     {

lib/Backend/NativeCodeGenerator.cpp

@@ -89,6 +89,8 @@ NativeCodeGenerator::~NativeCodeGenerator()
 {
     Assert(this->IsClosed());
 
+    DelayDeletingFunctionTable::Clear();
+
 #ifdef PROFILE_EXEC
     if (this->foregroundCodeGenProfiler != nullptr)
     {
@@ -892,6 +894,10 @@ void NativeCodeGenerator::CodeGen(PageAllocator* pageAllocator, CodeGenWorkItemI
 void
 NativeCodeGenerator::CodeGen(PageAllocator * pageAllocator, CodeGenWorkItem* workItem, const bool foreground)
 {
+    // flush the function tables before allocating any new  code
+    // to prevent old function table is referring to new code address
+    DelayDeletingFunctionTable::Clear();
+
     if(foreground)
     {
         // Func::Codegen has a lot of things on the stack, so probe the stack here instead
@@ -3200,14 +3206,14 @@ NativeCodeGenerator::EnterScriptStart()
 }
 
 void
-FreeNativeCodeGenAllocation(Js::ScriptContext *scriptContext, Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress)
+FreeNativeCodeGenAllocation(Js::ScriptContext *scriptContext, Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress, void** functionTable)
 {
     if (!scriptContext->GetNativeCodeGenerator())
-    {
+    { 
         return;
     }
 
-    scriptContext->GetNativeCodeGenerator()->QueueFreeNativeCodeGenAllocation((void*)codeAddress, (void*)thunkAddress);
+    scriptContext->GetNativeCodeGenerator()->QueueFreeNativeCodeGenAllocation((void*)codeAddress, (void*)thunkAddress, functionTable);
 }
 
 bool TryReleaseNonHiPriWorkItem(Js::ScriptContext* scriptContext, CodeGenWorkItem* workItem)
@@ -3238,22 +3244,30 @@ bool NativeCodeGenerator::TryReleaseNonHiPriWorkItem(CodeGenWorkItem* workItem)
 }
 
 void
-NativeCodeGenerator::FreeNativeCodeGenAllocation(void* codeAddress)
+NativeCodeGenerator::FreeNativeCodeGenAllocation(void* codeAddress, void** functionTable)
 {
     if (JITManager::GetJITManager()->IsOOPJITEnabled())
     {
+        // function table delete in content process
+#if PDATA_ENABLED && defined(_WIN32)
+        if (functionTable && *functionTable)
+        {
+            NtdllLibrary::Instance->DeleteGrowableFunctionTable(*functionTable);
+            *functionTable = nullptr;
+        }
+#endif
         ThreadContext * context = this->scriptContext->GetThreadContext();
         HRESULT hr = JITManager::GetJITManager()->FreeAllocation(context->GetRemoteThreadContextAddr(), (intptr_t)codeAddress);
         JITManager::HandleServerCallResult(hr, RemoteCallType::MemFree);
     }
     else if(this->backgroundAllocators)
     {
-        this->backgroundAllocators->emitBufferManager.FreeAllocation(codeAddress);
+        this->backgroundAllocators->emitBufferManager.FreeAllocation(codeAddress, functionTable);
     }
 }
 
 void
-NativeCodeGenerator::QueueFreeNativeCodeGenAllocation(void* codeAddress, void * thunkAddress)
+NativeCodeGenerator::QueueFreeNativeCodeGenAllocation(void* codeAddress, void * thunkAddress, void** functionTable)
 {
     ASSERT_THREAD();
 
@@ -3283,24 +3297,24 @@ NativeCodeGenerator::QueueFreeNativeCodeGenAllocation(void* codeAddress, void *
     // OOP JIT will always queue a job
 
     // The foreground allocators may have been used
-    if(this->foregroundAllocators && this->foregroundAllocators->emitBufferManager.FreeAllocation(codeAddress))
+    if (this->foregroundAllocators && this->foregroundAllocators->emitBufferManager.FreeAllocation(codeAddress, functionTable))
     {
         return;
     }
 
     // The background allocators were used. Queue a job to free the allocation from the background thread.
-    this->freeLoopBodyManager.QueueFreeLoopBodyJob(codeAddress, thunkAddress);
+    this->freeLoopBodyManager.QueueFreeLoopBodyJob(codeAddress, thunkAddress, functionTable);
 }
 
-void NativeCodeGenerator::FreeLoopBodyJobManager::QueueFreeLoopBodyJob(void* codeAddress, void * thunkAddress)
+void NativeCodeGenerator::FreeLoopBodyJobManager::QueueFreeLoopBodyJob(void* codeAddress, void * thunkAddress, void** functionTable)
 {
     Assert(!this->isClosed);
 
-    FreeLoopBodyJob* job = HeapNewNoThrow(FreeLoopBodyJob, this, codeAddress, thunkAddress);
+    FreeLoopBodyJob* job = HeapNewNoThrow(FreeLoopBodyJob, this, codeAddress, thunkAddress, *functionTable);
 
     if (job == nullptr)
     {
-        FreeLoopBodyJob stackJob(this, codeAddress, thunkAddress, false /* heapAllocated */);
+        FreeLoopBodyJob stackJob(this, codeAddress, thunkAddress, *functionTable, false /* heapAllocated */);
 
         {
             AutoOptionalCriticalSection lock(Processor()->GetCriticalSection());
@@ -3324,6 +3338,9 @@ void NativeCodeGenerator::FreeLoopBodyJobManager::QueueFreeLoopBodyJob(void* cod
             HeapDelete(job);
         }
     }
+
+    // function table successfully transferred to background job
+    *functionTable = nullptr;
 }
 
 #ifdef PROFILE_EXEC

lib/Backend/NativeCodeGenerator.h

@@ -104,10 +104,10 @@ public:
     void UpdateQueueForDebugMode();
     bool IsBackgroundJIT() const;
     void EnterScriptStart();
-    void FreeNativeCodeGenAllocation(void* codeAddress);
+    void FreeNativeCodeGenAllocation(void* codeAddress, void** functionTable);
     bool TryReleaseNonHiPriWorkItem(CodeGenWorkItem* workItem);
 
-    void QueueFreeNativeCodeGenAllocation(void* codeAddress, void* thunkAddress);
+    void QueueFreeNativeCodeGenAllocation(void* codeAddress, void* thunkAddress, void** functionTable);
 
     bool IsClosed() { return isClosed; }
     void AddWorkItem(CodeGenWorkItem* workItem);
@@ -201,10 +201,11 @@ private:
     class FreeLoopBodyJob: public JsUtil::Job
     {
     public:
-        FreeLoopBodyJob(JsUtil::JobManager *const manager, void* codeAddress, void* thunkAddress, bool isHeapAllocated = true):
+        FreeLoopBodyJob(JsUtil::JobManager *const manager, void* codeAddress, void* thunkAddress, void* functionTable, bool isHeapAllocated = true):
           JsUtil::Job(manager),
           codeAddress(codeAddress),
           thunkAddress(thunkAddress),
+          functionTable(functionTable),
           heapAllocated(isHeapAllocated)
         {
         }
@@ -212,6 +213,7 @@ private:
         bool heapAllocated;
         void* codeAddress;
         void* thunkAddress;
+        void* functionTable;
     };
 
     class FreeLoopBodyJobManager sealed: public WaitableJobManager
@@ -279,8 +281,9 @@ private:
         {
             FreeLoopBodyJob* freeLoopBodyJob = static_cast<FreeLoopBodyJob*>(job);
 
+            void* functionTable = freeLoopBodyJob->functionTable;
             // Free Loop Body
-            nativeCodeGen->FreeNativeCodeGenAllocation(freeLoopBodyJob->codeAddress);
+            nativeCodeGen->FreeNativeCodeGenAllocation(freeLoopBodyJob->codeAddress, &functionTable);
 
             return true;
         }
@@ -303,7 +306,7 @@ private:
             }
         }
 
-        void QueueFreeLoopBodyJob(void* codeAddress, void* thunkAddress);
+        void QueueFreeLoopBodyJob(void* codeAddress, void* thunkAddress, void** functionTable);
 
     private:
         NativeCodeGenerator* nativeCodeGen;

lib/Backend/PDataManager.cpp

@@ -45,6 +45,7 @@ void PDataManager::UnregisterPdata(RUNTIME_FUNCTION* pdata)
 {
     if (AutoSystemInfo::Data.IsWin8OrLater())
     {
+        // TODO: need to move to background?
         NtdllLibrary::Instance->DeleteGrowableFunctionTable(pdata);
     }
     else

lib/Common/BackendApi.h

@@ -62,7 +62,7 @@ void UpdateNativeCodeGeneratorForDebugMode(NativeCodeGenerator* nativeCodeGen);
 CriticalSection *GetNativeCodeGenCriticalSection(NativeCodeGenerator *pNativeCodeGen);
 bool TryReleaseNonHiPriWorkItem(Js::ScriptContext* scriptContext, CodeGenWorkItem* workItem);
 void NativeCodeGenEnterScriptStart(NativeCodeGenerator * nativeCodeGen);
-void FreeNativeCodeGenAllocation(Js::ScriptContext* scriptContext, Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress);
+void FreeNativeCodeGenAllocation(Js::ScriptContext* scriptContext, Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress, void** functionTable);
 InProcCodeGenAllocators* GetForegroundAllocator(NativeCodeGenerator * nativeCodeGen, PageAllocator* pageallocator);
 void GenerateFunction(NativeCodeGenerator * nativeCodeGen, Js::FunctionBody * functionBody, Js::ScriptFunction * function = NULL);
 void GenerateLoopBody(NativeCodeGenerator * nativeCodeGen, Js::FunctionBody * functionBody, Js::LoopHeader * loopHeader, Js::EntryPointInfo* entryPointInfo, uint localCount, Js::Var localSlots[]);

lib/Common/Common/Jobs.cpp

@@ -27,6 +27,7 @@
 #include "Common/Jobs.inl"
 #include "Core/CommonMinMax.h"
 #include "Memory/RecyclerWriteBarrierManager.h"
+#include "Memory/XDataAllocator.h"
 
 namespace JsUtil
 {
@@ -614,7 +615,8 @@ namespace JsUtil
         threadId(GetCurrentThreadContextId()),
         threadService(threadService),
         threadCount(0),
-        maxThreadCount(0)
+        maxThreadCount(0),
+        hasExtraWork(false)
     {
         if (!threadService->HasCallback())
         {
@@ -676,6 +678,8 @@ namespace JsUtil
         //Wait for 1 sec on jobReady and shutdownBackgroundThread events.
         unsigned int result = WaitForMultipleObjectsEx(_countof(handles), handles, false, 1000, false);
 
+        DoExtraWork();
+
         while (result == WAIT_TIMEOUT)
         {
             if (threadData->CanDecommit())
@@ -685,6 +689,7 @@ namespace JsUtil
                 this->ForEachManager([&](JobManager *manager){
                     manager->OnDecommit(threadData);
                 });
+
                 result = WaitForMultipleObjectsEx(_countof(handles), handles, false, INFINITE, false);
             }
             else
@@ -701,6 +706,15 @@ namespace JsUtil
         return result == WAIT_OBJECT_0;
     }
 
+    void BackgroundJobProcessor::DoExtraWork()
+    {
+        while (hasExtraWork)
+        {
+            DelayDeletingFunctionTable::Clear();
+            Sleep(50);
+        }        
+    }
+
     bool BackgroundJobProcessor::WaitWithThreadForThreadStartedOrClosingEvent(ParallelThreadData *parallelThreadData, const unsigned int milliseconds)
     {
         return WaitWithThread(parallelThreadData, parallelThreadData->threadStartedOrClosing, milliseconds);
@@ -1103,6 +1117,9 @@ namespace JsUtil
             }
             criticalSection.Leave();
 
+            // flush the function tables in background thread after closed and before shutting down thread
+            DelayDeletingFunctionTable::Clear();
+
             EDGE_ETW_INTERNAL(EventWriteJSCRIPT_NATIVECODEGEN_STOP(this, 0));
         }
     }
@@ -1403,6 +1420,18 @@ namespace JsUtil
 #endif
     }
 
+    void BackgroundJobProcessor::StartExtraWork()
+    {
+        hasExtraWork = true;
+
+        // Signal the background thread to wake up and process the extra work.
+        jobReady.Set();
+    }
+    void BackgroundJobProcessor::EndExtraWork()
+    {
+        hasExtraWork = false;
+    }
+
 #if DBG_DUMP
     //Just for debugging purpose
     char16 const * const  BackgroundJobProcessor::DebugThreadNames[16] = {

lib/Common/Common/Jobs.h

@@ -360,6 +360,9 @@ namespace JsUtil
     public:
         // Closes the job processor and closes the handle of background threads.
         virtual void Close();
+
+        virtual void StartExtraWork() { };
+        virtual void EndExtraWork() { };
     };
 
     // -------------------------------------------------------------------------------------------------------------------------
@@ -457,6 +460,8 @@ namespace JsUtil
         unsigned int maxThreadCount;
         ParallelThreadData **parallelThreadData;
 
+        bool hasExtraWork;
+
 #if DBG_DUMP
         static  char16 const * const  DebugThreadNames[16];
 #endif
@@ -465,6 +470,8 @@ namespace JsUtil
         BackgroundJobProcessor(AllocationPolicyManager* policyManager, ThreadService *threadService, bool disableParallelThreads);
         ~BackgroundJobProcessor();
 
+        virtual void StartExtraWork() override;
+        virtual void EndExtraWork() override;
 
     private:
         bool WaitWithThread(ParallelThreadData *parallelThreadData, const Event &e, const unsigned int milliseconds = INFINITE);
@@ -482,6 +489,8 @@ namespace JsUtil
         void InitializeParallelThreadData(AllocationPolicyManager* policyManager, bool disableParallelThreads);
         void InitializeParallelThreadDataForThreadServiceCallBack(AllocationPolicyManager* policyManager);
 
+        void DoExtraWork();
+
     public:
         virtual void AddManager(JobManager *const manager) override;
         virtual void RemoveManager(JobManager *const manager) override;

lib/Common/Core/DelayLoadLibrary.cpp

@@ -70,6 +70,7 @@ DWORD NtdllLibrary::AddGrowableFunctionTable( _Out_ PVOID * DynamicTable,
     _In_ ULONG_PTR RangeBase,
     _In_ ULONG_PTR RangeEnd )
 {
+    Assert(AutoSystemInfo::Data.IsWin8OrLater());
     if(m_hModule)
     {
         if(addGrowableFunctionTable == NULL)
@@ -93,6 +94,7 @@ DWORD NtdllLibrary::AddGrowableFunctionTable( _Out_ PVOID * DynamicTable,
 
 VOID NtdllLibrary::DeleteGrowableFunctionTable( _In_ PVOID DynamicTable )
 {
+    Assert(AutoSystemInfo::Data.IsWin8OrLater());
     if(m_hModule)
     {
         if(deleteGrowableFunctionTable == NULL)

lib/Common/Memory/CMakeLists.txt

@@ -3,6 +3,7 @@ set (CCM_SOURCE_FILES ${CCM_SOURCE_FILES}
     ArenaAllocator.cpp
     CustomHeap.cpp
     CommonMemoryPch.cpp
+    DelayDeletingFunctionTable.cpp
     EtwMemoryTracking.cpp
     ForcedMemoryConstraints.cpp
     HeapAllocator.cpp

lib/Common/Memory/Chakra.Common.Memory.vcxproj

@@ -90,6 +90,7 @@
       <PrecompiledHeader>Create</PrecompiledHeader>
     </ClCompile>
     <ClCompile Include="$(MSBuildThisFileDirectory)SectionAllocWrapper.cpp" />
+    <ClCompile Include="DelayDeletingFunctionTable.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="AllocationPolicyManager.h" />
@@ -186,4 +187,4 @@
     <Import Project="$(VCTargetsPath)\BuildCustomizations\masm.targets" />
     <Import Project="$(BuildConfig_ARMASM_Path)armasm.targets" />
   </ImportGroup>
-</Project>
+</Project>

lib/Common/Memory/Chakra.Common.Memory.vcxproj.filters

@@ -54,7 +54,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)arm64\XDataAllocator.cpp">
       <Filter>arm64</Filter>
     </ClCompile>
-    <ClCompile Include="SectionAllocWrapper.cpp" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)SectionAllocWrapper.cpp" />
+    <ClCompile Include="DelayDeletingFunctionTable.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="Allocator.h" />

lib/Common/Memory/CustomHeap.cpp

@@ -3,16 +3,17 @@
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------
 #include "CommonMemoryPch.h"
-#ifdef _M_X64
-#include "Memory/amd64/XDataAllocator.h"
-#elif defined(_M_ARM)
-#include "Memory/arm/XDataAllocator.h"
+#include "Memory/XDataAllocator.h"
+#if defined(_M_ARM)
 #include <wchar.h>
-#elif defined(_M_ARM64)
-#include "Memory/arm64/XDataAllocator.h"
 #endif
 #include "CustomHeap.h"
 
+#if PDATA_ENABLED && defined(_WIN32)
+#include "Core/DelayLoadLibrary.h"
+#include <malloc.h>
+#endif
+
 namespace Memory
 {
 namespace CustomHeap

lib/Common/Memory/DelayDeletingFunctionTable.cpp

@@ -0,0 +1,82 @@
+//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+
+#include "CommonMemoryPch.h"
+#include "Memory/XDataAllocator.h"
+#if PDATA_ENABLED && defined(_WIN32)
+#include "Core/DelayLoadLibrary.h"
+#include <malloc.h>
+#endif
+
+PSLIST_HEADER DelayDeletingFunctionTable::Head = nullptr;
+DelayDeletingFunctionTable DelayDeletingFunctionTable::Instance;
+
+DelayDeletingFunctionTable::DelayDeletingFunctionTable()
+{
+#if PDATA_ENABLED && defined(_WIN32)
+    Head = (PSLIST_HEADER)_aligned_malloc(sizeof(SLIST_HEADER), MEMORY_ALLOCATION_ALIGNMENT);
+    if (Head)
+    {
+        InitializeSListHead(Head);
+    }
+#endif
+}
+DelayDeletingFunctionTable::~DelayDeletingFunctionTable()
+{
+#if PDATA_ENABLED && defined(_WIN32)
+    Clear();
+    if (Head)
+    {
+        DebugOnly(SLIST_ENTRY* entry = InterlockedPopEntrySList(Head));
+        Assert(entry == nullptr);
+        _aligned_free(Head);
+        Head = nullptr;
+    }
+#endif
+}
+
+
+bool DelayDeletingFunctionTable::AddEntry(FunctionTableHandle ft)
+{
+#if PDATA_ENABLED && defined(_WIN32)
+    if (Head)
+    {
+        FunctionTableNode* node = (FunctionTableNode*)_aligned_malloc(sizeof(FunctionTableNode), MEMORY_ALLOCATION_ALIGNMENT);
+        if (node)
+        {
+            node->functionTable = ft;
+            InterlockedPushEntrySList(Head, &(node->itemEntry));
+            return true;
+        }
+    }
+#endif
+    return false;
+}
+
+void DelayDeletingFunctionTable::Clear()
+{
+#if PDATA_ENABLED && defined(_WIN32)
+    if (Head)
+    {
+        SLIST_ENTRY* entry = InterlockedPopEntrySList(Head);
+        while (entry)
+        {
+            FunctionTableNode* list = (FunctionTableNode*)entry;
+            NtdllLibrary::Instance->DeleteGrowableFunctionTable(list->functionTable);
+            _aligned_free(entry);
+            entry = InterlockedPopEntrySList(Head);
+        }
+    }
+#endif
+}
+
+bool DelayDeletingFunctionTable::IsEmpty()
+{
+#if PDATA_ENABLED && defined(_WIN32)
+    return QueryDepthSList(Head) == 0;
+#else
+    return true;
+#endif
+}

lib/Common/Memory/XDataAllocator.h

@@ -11,3 +11,25 @@
 #elif defined(_M_ARM64)
 #include "Memory/arm64/XDataAllocator.h"
 #endif
+
+struct FunctionTableNode
+{
+#ifdef _WIN32
+    SLIST_ENTRY itemEntry;
+    FunctionTableHandle functionTable;
+
+#endif
+};
+
+struct DelayDeletingFunctionTable
+{
+    static PSLIST_HEADER Head;
+    static DelayDeletingFunctionTable Instance;
+
+    DelayDeletingFunctionTable();
+    ~DelayDeletingFunctionTable();
+
+    static bool AddEntry(FunctionTableHandle ft);
+    static void Clear();
+    static bool IsEmpty();
+};

lib/Common/Memory/amd64/XDataAllocator.cpp

@@ -162,12 +162,7 @@ void XDataAllocator::Register(XDataAllocation * xdataInfo, ULONG_PTR functionSta
 void XDataAllocator::Unregister(XDataAllocation * xdataInfo)
 {
 #ifdef _WIN32
-    // Delete the table
-    if (AutoSystemInfo::Data.IsWin8OrLater())
-    {
-        NtdllLibrary::Instance->DeleteGrowableFunctionTable(xdataInfo->functionTable);
-    }
-    else
+    if (!AutoSystemInfo::Data.IsWin8OrLater())
     {
         BOOLEAN success = RtlDeleteFunctionTable(&xdataInfo->pdata);
         Assert(success);

lib/Common/Memory/amd64/XDataAllocator.h

@@ -18,6 +18,9 @@ namespace Memory
 struct XDataAllocation : public SecondaryAllocation
 {
     XDataAllocation()
+#ifdef _WIN32
+        :functionTable(nullptr)
+#endif
     {}
 
     bool IsFreed() const

lib/Common/Memory/arm/XDataAllocator.cpp

@@ -99,9 +99,7 @@ void XDataAllocator::Register(XDataAllocation * xdataInfo, DWORD functionStart,
 /* static */
 void XDataAllocator::Unregister(XDataAllocation * xdataInfo)
 {
-#ifdef _WIN32
-    NtdllLibrary::Instance->DeleteGrowableFunctionTable(xdataInfo->functionTable);
-#else  // !_WIN32
+#ifndef _WIN32
     Assert(ReadHead(xdataInfo->address));  // should be non-empty .eh_frame
     __DEREGISTER_FRAME(xdataInfo->address);
 #endif

lib/Common/Memory/arm64/XDataAllocator.cpp

@@ -154,10 +154,7 @@ void XDataAllocator::Register(XDataAllocation * xdataInfo, ULONG_PTR functionSta
 /* static */
 void XDataAllocator::Unregister(XDataAllocation * xdataInfo)
 {
-#ifdef _WIN32
-    // Delete the table
-    NtdllLibrary::Instance->DeleteGrowableFunctionTable(xdataInfo->functionTable);
-#else  // !_WIN32
+#ifndef _WIN32
     Assert(ReadHead(xdataInfo->address));  // should be non-empty .eh_frame
     __DEREGISTER_FRAME(xdataInfo->address);
 #endif

lib/Common/Memory/arm64/XDataAllocator.h

@@ -19,6 +19,9 @@ namespace Memory
     struct XDataAllocation : public SecondaryAllocation
     {
         XDataAllocation()
+#ifdef _WIN32
+            :functionTable(nullptr)
+#endif
         {}
 
         bool IsFreed() const

lib/JITServer/JITServer.cpp

@@ -672,7 +672,7 @@ ServerFreeAllocation(
 
     return ServerCallWrapper(context, [&]()->HRESULT
     {
-        context->GetCodeGenAllocators()->emitBufferManager.FreeAllocation((void*)codeAddress);
+        context->GetCodeGenAllocators()->emitBufferManager.FreeAllocation((void*)codeAddress, nullptr);
         return S_OK;
     });
 }

lib/Runtime/Base/FunctionBody.cpp

@@ -8846,31 +8846,38 @@ namespace Js
     {
         if (this->GetState() != CleanedUp)
         {
-            // Unregister xdataInfo before OnCleanup() which may release xdataInfo->address
 #if ENABLE_NATIVE_CODEGEN
-#if defined(TARGET_64)
-            if (this->xdataInfo != nullptr)
-            {
-                XDataAllocator::Unregister(this->xdataInfo);
-                HeapDelete(this->xdataInfo);
-                this->xdataInfo = nullptr;
-            }
-#elif defined(_M_ARM)
+            void* functionTable = nullptr;
+#if PDATA_ENABLED
             if (this->xdataInfo != nullptr)
             {
+#ifdef _WIN32
+                functionTable = this->xdataInfo->functionTable;
+#endif
                 XDataAllocator::Unregister(this->xdataInfo);
+#if defined(_M_ARM32_OR_ARM64)
                 if (JITManager::GetJITManager()->IsOOPJITEnabled())
+#endif
                 {
                     HeapDelete(this->xdataInfo);
                 }
                 this->xdataInfo = nullptr;
             }
-#endif
 #endif
 
-            this->OnCleanup(isShutdown);
+            this->OnCleanup(isShutdown, &functionTable);
+
+#if PDATA_ENABLED && defined(_WIN32)
+            // functionTable is not transferred somehow, delete in-thread
+            if (functionTable)
+            {
+                if (!DelayDeletingFunctionTable::AddEntry(functionTable))
+                {
+                    NtdllLibrary::Instance->DeleteGrowableFunctionTable(functionTable);
+                }
+            }
+#endif
 
-#if ENABLE_NATIVE_CODEGEN
             FreeJitTransferData();
 
             if (this->bailoutRecordMap != nullptr)
@@ -8992,7 +8999,15 @@ namespace Js
         // Reset the entry point upon a lazy bailout.
         this->Reset(true);
         Assert(this->nativeAddress != nullptr);
-        FreeNativeCodeGenAllocation(GetScriptContext(), this->nativeAddress, this->thunkAddress);
+
+        void* functionTable = nullptr;
+#if PDATA_ENABLED && defined(_WIN32)
+        if (this->xdataInfo)
+        {
+            functionTable = this->xdataInfo->functionTable;
+        }
+#endif
+        FreeNativeCodeGenAllocation(GetScriptContext(), this->nativeAddress, this->thunkAddress, &functionTable);
         this->nativeAddress = nullptr;
         this->jsMethod = nullptr;
     }
@@ -9088,7 +9103,7 @@ namespace Js
         return functionProxy->GetFunctionBody();
     }
 
-    void FunctionEntryPointInfo::OnCleanup(bool isShutdown)
+    void FunctionEntryPointInfo::OnCleanup(bool isShutdown, void** functionTable)
     {
         if (this->IsCodeGenDone())
         {
@@ -9099,19 +9114,7 @@ namespace Js
                 HeapDelete(this->inlineeFrameMap);
                 this->inlineeFrameMap = nullptr;
             }
-#if PDATA_ENABLED
-            if (this->xdataInfo != nullptr)
-            {
-                XDataAllocator::Unregister(this->xdataInfo);
-#if defined(_M_ARM32_OR_ARM64)
-                if (JITManager::GetJITManager()->IsOOPJITEnabled())
-#endif
-                {
-                    HeapDelete(this->xdataInfo);
-                }
-                this->xdataInfo = nullptr;
-            }
-#endif
+
 #endif
 
             if(nativeEntryPointProcessed)
@@ -9162,7 +9165,8 @@ namespace Js
 
                 if (validationCookie == currentCookie)
                 {
-                    scriptContext->FreeFunctionEntryPoint((Js::JavascriptMethod)this->GetNativeAddress(), this->GetThunkAddress());
+                    scriptContext->FreeFunctionEntryPoint((Js::JavascriptMethod)this->GetNativeAddress(), this->GetThunkAddress(), functionTable);
+                    *functionTable = nullptr;
                 }
             }
 
@@ -9422,7 +9426,7 @@ namespace Js
 
     //End AsmJs Support
 
-    void LoopEntryPointInfo::OnCleanup(bool isShutdown)
+    void LoopEntryPointInfo::OnCleanup(bool isShutdown, void** functionTable)
     {
 #ifdef ASMJS_PLAT
         if (this->IsCodeGenDone() && !this->GetIsTJMode())
@@ -9439,19 +9443,6 @@ namespace Js
                 HeapDelete(this->inlineeFrameMap);
                 this->inlineeFrameMap = nullptr;
             }
-#if PDATA_ENABLED
-            if (this->xdataInfo != nullptr)
-            {
-                XDataAllocator::Unregister(this->xdataInfo);
-#if defined(_M_ARM32_OR_ARM64)
-                if (JITManager::GetJITManager()->IsOOPJITEnabled())
-#endif
-                {
-                    HeapDelete(this->xdataInfo);
-                }
-                this->xdataInfo = nullptr;
-            }
-#endif
 #endif
 
             if (!isShutdown)
@@ -9483,7 +9474,8 @@ namespace Js
 
                 if (validationCookie == currentCookie)
                 {
-                    scriptContext->FreeFunctionEntryPoint(reinterpret_cast<Js::JavascriptMethod>(this->GetNativeAddress()), this->GetThunkAddress());
+                    scriptContext->FreeFunctionEntryPoint(reinterpret_cast<Js::JavascriptMethod>(this->GetNativeAddress()), this->GetThunkAddress(), functionTable);
+                    *functionTable = nullptr;
                 }
             }
 

lib/Runtime/Base/FunctionBody.h

@@ -485,7 +485,7 @@ namespace Js
 
         virtual void ReleasePendingWorkItem() {};
 
-        virtual void OnCleanup(bool isShutdown) = 0;
+        virtual void OnCleanup(bool isShutdown, void** functionTable) = 0;
 
 #ifdef PERF_COUNTERS
         virtual void OnRecorded() = 0;
@@ -907,7 +907,7 @@ namespace Js
         }
 #endif
 
-        virtual void OnCleanup(bool isShutdown) override;
+        virtual void OnCleanup(bool isShutdown, void** functionTable) override;
 
         virtual void ReleasePendingWorkItem() override;
 
@@ -936,7 +936,7 @@ namespace Js
 
         virtual FunctionBody *GetFunctionBody() const override;
 
-        virtual void OnCleanup(bool isShutdown) override;
+        virtual void OnCleanup(bool isShutdown, void** functionTable) override;
 
 #if ENABLE_NATIVE_CODEGEN
         virtual void ResetOnNativeCodeInstallFailure() override;

lib/Runtime/Base/ScriptContext.cpp

@@ -654,6 +654,24 @@ namespace Js
 
                 if (hasFunctions)
                 {
+#if ENABLE_NATIVE_CODEGEN
+                    struct AutoReset
+                    {
+                        AutoReset(ThreadContext* threadContext)
+                            :threadContext(threadContext)
+                        {
+                            // indicate background thread that we need help to delete the xData
+                            threadContext->GetJobProcessor()->StartExtraWork();
+                        }
+                        ~AutoReset()
+                        {
+                            threadContext->GetJobProcessor()->EndExtraWork();
+                        }
+
+                        ThreadContext* threadContext;
+                    } autoReset(this->GetThreadContext());
+#endif
+
                     // We still need to walk through all the function bodies and call cleanup
                     // because otherwise ETW events might not get fired if a GC doesn't happen
                     // and the thread context isn't shut down cleanly (process detach case)
@@ -4401,10 +4419,10 @@ namespace Js
     }
 #endif
 
-    void ScriptContext::FreeFunctionEntryPoint(Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress)
+    void ScriptContext::FreeFunctionEntryPoint(Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress, void** functionTable)
     {
 #if ENABLE_NATIVE_CODEGEN
-        FreeNativeCodeGenAllocation(this, codeAddress, thunkAddress);
+        FreeNativeCodeGenAllocation(this, codeAddress, thunkAddress, functionTable);
 #endif
     }
 

lib/Runtime/Base/ScriptContext.h

@@ -1438,7 +1438,7 @@ private:
             return threadContext->GetEmptyStringPropertyId();
         }
 
-        void FreeFunctionEntryPoint(Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress);
+        void FreeFunctionEntryPoint(Js::JavascriptMethod codeAddress, Js::JavascriptMethod thunkAddress, void** functionTable);
 
     private:
         uint CloneSource(Utf8SourceInfo* info);

lib/Runtime/Base/ThreadContext.cpp

@@ -628,7 +628,6 @@ void ThreadContext::CloseForJSRT()
     ShutdownThreads();
 }
 
-
 ThreadContext* ThreadContext::GetContextForCurrentThread()
 {
     ThreadContextTLSEntry * tlsEntry = ThreadContextTLSEntry::GetEntryForCurrentThread();