fix virtual alloc upper limit

Default `lpMaximumApplicationAddress` can be beyond what is allowed for the process.
Get AS/RSS limit before sending an allocation request.

Also, crash early in case of allocation fails.

lib/Common/Memory/RecyclerWriteBarrierManager.cpp

@@ -226,10 +226,28 @@ X64WriteBarrierCardTableManager::Initialize()
         // On Win8, reserving 32 GB is fine since reservations don't incur a cost. On Win7, the cost
         // of a reservation can be approximated as 2KB per MB of reserved size. In our case, we take
         // an overhead of 96KB for our card table.
-        const unsigned __int64 maxUmProcessAddressSpace = (__int64) AutoSystemInfo::Data.lpMaximumApplicationAddress;
+
+        // xplat: GetRLimit AS / RSS for ``the maximum size of the process's virtual memory``
+        size_t memoryLimit;
+        if (!PlatformAgnostic::SystemInfo::GetMaxVirtualMemory(&memoryLimit))
+        {
+            memoryLimit = (size_t) AutoSystemInfo::Data.lpMaximumApplicationAddress; // try upper limit
+        }
+        else
+        {
+            // Safest option : Max RSS can be beyond what we can allocate, aim the smaller one
+            memoryLimit = min(memoryLimit, (size_t) AutoSystemInfo::Data.lpMaximumApplicationAddress);
+        }
+        const unsigned __int64 maxUmProcessAddressSpace = (__int64) memoryLimit;
+
         _cardTableNumEntries = Math::Align<size_t>(maxUmProcessAddressSpace / AutoSystemInfo::PageSize, AutoSystemInfo::PageSize) /* s_writeBarrierPageSize */;
 
         LPVOID cardTableSpace = ::VirtualAlloc(NULL, _cardTableNumEntries, MEM_RESERVE, PAGE_READWRITE);
+        if (!cardTableSpace) // Crash Early with a meaningful message. Otherwise the behavior is undefined.
+        {
+            fprintf(stderr, "Out of Memory\n"); fflush(stderr);
+            abort();
+        }
 
         _cardTable = (BYTE*) cardTableSpace;
     }

lib/Common/PlatformAgnostic/SystemInfo.h

@@ -31,6 +31,8 @@ namespace PlatformAgnostic
             *totalRam = SystemInfo::data.totalRam;
             return true;
         }
+
+        static bool GetMaxVirtualMemory(size_t *totalAS);
     };
 } // namespace PlatformAgnostic
 

lib/Runtime/PlatformAgnostic/Platform/Linux/SystemInfo.cpp

@@ -6,6 +6,7 @@
 #include "Common.h"
 #include "ChakraPlatform.h"
 #include <sys/sysinfo.h>
+#include <sys/resource.h>
 
 namespace PlatformAgnostic
 {
@@ -23,4 +24,15 @@ namespace PlatformAgnostic
             totalRam = systemInfo.totalram;
         }
     }
+
+    bool SystemInfo::GetMaxVirtualMemory(size_t *totalAS)
+    {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_AS, &limit) != 0)
+        {
+            return false;
+        }
+        *totalAS = limit.rlim_cur;
+        return true;
+    }
 }

lib/Runtime/PlatformAgnostic/Platform/Unix/SystemInfo.cpp

@@ -22,4 +22,16 @@ namespace PlatformAgnostic
             totalRam = 0;
         }
     }
+
+    bool SystemInfo::GetMaxVirtualMemory(size_t *totalAS)
+    {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_AS, &limit) != 0)
+        {
+            return false;
+        }
+
+        *totalAS = limit.rlim_cur;
+        return true;
+    }
 }

lib/Runtime/PlatformAgnostic/Platform/Windows/SystemInfo.cpp

@@ -19,4 +19,13 @@ namespace PlatformAgnostic
             totalRam = static_cast<size_t>(ram) * 1024;
         }
     }
+
+    bool SystemInfo::GetMaxVirtualMemory(size_t *totalAS)
+    {
+        SYSTEM_INFO info;
+        GetSystemInfo(&info);
+        *totalAS = (size_t) info.lpMaximumApplicationAddress;
+        return true;
+    }
+
 }

pal/src/init/pal.cpp

@@ -94,7 +94,6 @@ static pthread_mutex_t init_critsec_mutex = PTHREAD_MUTEX_INITIALIZER;
    very first PAL_Initialize call, and is freed afterward. */
 static PCRITICAL_SECTION init_critsec = NULL;
 
-static BOOL INIT_IncreaseDescriptorLimit(void);
 static LPWSTR INIT_FormatCommandLine (int argc, const char * const *argv);
 static LPWSTR INIT_FindEXEPath(LPCSTR exe_name);
 
@@ -194,13 +193,6 @@ Initialize()
         }
 #endif  // _DEBUG
 
-        if (!INIT_IncreaseDescriptorLimit())
-        {
-            ERROR("Unable to increase the file descriptor limit!\n");
-            // We can continue if this fails; we'll just have problems if
-            // we use large numbers of threads or have many open files.
-        }
-
         /* initialize the shared memory infrastructure */
         if (!SHMInitialize())
         {
@@ -742,40 +734,6 @@ void PALInitUnlock(void)
     InternalLeaveCriticalSection(pThread, init_critsec);
 }
 
-/* Internal functions *********************************************************/
-
-/*++
-Function:
-    INIT_IncreaseDescriptorLimit [internal]
-
-Abstract:
-    Calls setrlimit(2) to increase the maximum number of file descriptors
-    this process can open.
-
-Return value:
-    TRUE if the call to setrlimit succeeded; FALSE otherwise.
---*/
-static BOOL INIT_IncreaseDescriptorLimit(void)
-{
-    struct rlimit rlp;
-    int result;
-
-    result = getrlimit(RLIMIT_NOFILE, &rlp);
-    if (result != 0)
-    {
-        return FALSE;
-    }
-    // Set our soft limit for file descriptors to be the same
-    // as the max limit.
-    rlp.rlim_cur = rlp.rlim_max;
-    result = setrlimit(RLIMIT_NOFILE, &rlp);
-    if (result != 0)
-    {
-        return FALSE;
-    }
-
-    return TRUE;
-}
 
 /*++
 Function: