Change GrowMemory to try to return nullptr instead of throwing out of memory.

Handle case where we fail to create the WebAssemblyMemory because we ran out of memory

lib/Parser/rterrors.h

@@ -394,6 +394,7 @@ RT_ERROR_MSG(WASMERR_NeedResponse, 7020, "%s is not a Reponse", "Response expect
 RT_ERROR_MSG(WASMERR_CantDetach, 7021, "", "Not allowed to detach WebAssembly.Memory buffer", kjstTypeError, 0)
 RT_ERROR_MSG(WASMERR_BufferGrowOnly, 7022, "", "WebAssembly.Memory can only grow", kjstTypeError, 0)
 RT_ERROR_MSG(WASMERR_LinkSignatureMismatch, 7023, "Cannot link import %s in link table due to a signature mismatch", "Function called with invalid signature", kjstWebAssemblyRuntimeError, 0)
+RT_ERROR_MSG(WASMERR_MemoryCreateFailed, 7024, "", "Failed to create WebAssembly.Memory", kjstTypeError, 0)
 
 // Wabt Errors
 RT_ERROR_MSG(WABTERR_WabtError, 7200, "%s", "Wabt Error.", kjstTypeError, 0)

lib/Runtime/Library/ArrayBuffer.cpp

@@ -895,7 +895,9 @@ namespace Js
         return newArrayBuffer;
     }
 
-    void JavascriptArrayBuffer::ReportDifferentialAllocation(uint32 newBufferLength)
+
+    template<typename Func>
+    void Js::JavascriptArrayBuffer::ReportDifferentialAllocation(uint32 newBufferLength, Func reportFailureFn)
     {
         Recycler* recycler = this->GetRecycler();
 
@@ -912,7 +914,7 @@ namespace Js
                     recycler->CollectNow<CollectOnTypedArrayAllocation>();
                     if (!recycler->ReportExternalMemoryAllocation(newBufferLength - this->bufferLength))
                     {
-                        JavascriptError::ThrowOutOfMemoryError(GetScriptContext());
+                        reportFailureFn();
                     }
                 }
             }
@@ -924,6 +926,15 @@ namespace Js
         }
     }
 
+
+    void JavascriptArrayBuffer::ReportDifferentialAllocation(uint32 newBufferLength)
+    {
+        ScriptContext* scriptContext = GetScriptContext();
+        ReportDifferentialAllocation(newBufferLength, [scriptContext] {
+            JavascriptError::ThrowOutOfMemoryError(scriptContext);
+        });
+    }
+
 #if ENABLE_TTD
     TTD::NSSnapObjects::SnapObjectType JavascriptArrayBuffer::GetSnapTag_TTD() const
     {
@@ -1027,22 +1038,30 @@ namespace Js
     {
         if (newBufferLength <= this->bufferLength)
         {
+            Assert(UNREACHED);
             JavascriptError::ThrowTypeError(GetScriptContext(), WASMERR_BufferGrowOnly);
         }
 
+        bool failedReport = false;
+        const auto reportFailedFn = [&failedReport] { failedReport = true; };
+
         WebAssemblyArrayBuffer* newArrayBuffer = nullptr;
 #if ENABLE_FAST_ARRAYBUFFER
         if (CONFIG_FLAG(WasmFastArray))
         {
-            ReportDifferentialAllocation(newBufferLength);
             AssertOrFailFast(this->buffer);
+            ReportDifferentialAllocation(newBufferLength, reportFailedFn);
+            if (failedReport)
+            {
+                return nullptr;
+            }
 
             LPVOID newMem = VirtualAlloc(this->buffer + this->bufferLength, newBufferLength - this->bufferLength, MEM_COMMIT, PAGE_READWRITE);
             if (!newMem)
             {
                 Recycler* recycler = this->GetRecycler();
                 recycler->ReportExternalMemoryFailure(newBufferLength);
-                JavascriptError::ThrowOutOfMemoryError(GetScriptContext());
+                return nullptr;
             }
             newArrayBuffer = GetLibrary()->CreateWebAssemblyArrayBuffer(this->buffer, newBufferLength);
         }
@@ -1051,26 +1070,26 @@ namespace Js
         if (this->GetByteLength() == 0)
         {
             newArrayBuffer = GetLibrary()->CreateWebAssemblyArrayBuffer(newBufferLength);
-            if (!newArrayBuffer->GetByteLength())
-            {
-                JavascriptError::ThrowOutOfMemoryError(GetScriptContext());
-            }
         }
         else
         {
-            ReportDifferentialAllocation(newBufferLength);
+            ReportDifferentialAllocation(newBufferLength, reportFailedFn);
+            if (failedReport)
+            {
+                return nullptr;
+            }
             byte* newBuffer = ReallocZero(this->buffer, this->bufferLength, newBufferLength);
             if (!newBuffer)
             {
                 this->GetRecycler()->ReportExternalMemoryFailure(newBufferLength - this->bufferLength);
-                JavascriptError::ThrowOutOfMemoryError(GetScriptContext());
+                return nullptr;
             }
             newArrayBuffer = GetLibrary()->CreateWebAssemblyArrayBuffer(newBuffer, newBufferLength);
         }
 
-        if (!newArrayBuffer)
+        if (!newArrayBuffer || !newArrayBuffer->GetByteLength())
         {
-            JavascriptError::ThrowOutOfMemoryError(GetScriptContext());
+            return nullptr;
         }
 
         AutoDiscardPTR<Js::ArrayBufferDetachedStateBase> state(DetachAndGetState());

lib/Runtime/Library/ArrayBuffer.h

@@ -260,6 +260,8 @@ namespace Js
 
         virtual ArrayBuffer * TransferInternal(DECLSPEC_GUARD_OVERFLOW uint32 newBufferLength) override;
 
+        template<typename Func>
+        void ReportDifferentialAllocation(uint32 newBufferLength, Func reportFailureFn);
         void ReportDifferentialAllocation(uint32 newBufferLength);
 
     protected:

lib/Runtime/Library/WebAssemblyInstance.cpp

@@ -463,6 +463,10 @@ void WebAssemblyInstance::ValidateTableAndMemory(WebAssemblyModule * wasmModule,
     else
     {
         mem = wasmModule->CreateMemory();
+        if (mem == nullptr)
+        {
+            JavascriptError::ThrowWebAssemblyLinkError(ctx, WASMERR_MemoryCreateFailed);
+        }
         env->SetMemory(0, mem);
     }
     ArrayBuffer * buffer = mem->GetBuffer();

lib/Runtime/Library/WebAssemblyMemory.cpp

@@ -151,6 +151,10 @@ WebAssemblyMemory::GrowInternal(uint32 deltaPages)
     try
     {
         newBuffer = m_buffer->GrowMemory(newBytes);
+        if (newBuffer == nullptr)
+        {
+            return -1;
+        }
     }
     catch (const JavascriptException& err)
     {
@@ -159,7 +163,6 @@ WebAssemblyMemory::GrowInternal(uint32 deltaPages)
         return -1;
     }
 
-    AssertOrFailFast(newBuffer);
     m_buffer = newBuffer;
     CompileAssert(ArrayBuffer::MaxArrayBufferLength / WebAssembly::PageSize <= INT32_MAX);
     return (int32)oldPageCount;
@@ -194,8 +197,15 @@ WebAssemblyMemory::EntryGetterBuffer(RecyclableObject* function, CallInfo callIn
 WebAssemblyMemory *
 WebAssemblyMemory::CreateMemoryObject(uint32 initial, uint32 maximum, ScriptContext * scriptContext)
 {
+    // This shouldn't overflow since we checked in the module, but just to be safe
     uint32 byteLength = UInt32Math::Mul<WebAssembly::PageSize>(initial);
     WebAssemblyArrayBuffer* buffer = scriptContext->GetLibrary()->CreateWebAssemblyArrayBuffer(byteLength);
+    Assert(buffer);
+    if (byteLength > 0 && buffer->GetByteLength() == 0)
+    {
+        // Failed to allocate buffer
+        return nullptr;
+    }
     return RecyclerNewFinalized(scriptContext->GetRecycler(), WebAssemblyMemory, buffer, initial, maximum, scriptContext->GetLibrary()->GetWebAssemblyMemoryType());
 }