Add compression and decompression utility and use it to optionally compress the parser state cache

Adds a new common utility class `CompressionUtilities` with two static methods:

```javascript
        static HRESULT CompressBuffer(
            _In_ ArenaAllocator* alloc,
            _In_ const byte* inputBuffer,
            _In_ size_t inputBufferByteCount,
            _Out_ byte** compressedBuffer,
            _Out_ size_t* compressedBufferByteCount,
            _In_opt_ CompressionAlgorithm algorithm = CompressionAlgorithm_Xpress);

        static HRESULT DecompressBuffer(
            _In_ ArenaAllocator* alloc,
            _In_ const byte* compressedBuffer,
            _In_ size_t compressedBufferByteCount,
            _Out_ byte** decompressedBuffer,
            _Out_ size_t* decompressedBufferByteCount,
            _In_opt_ CompressionAlgorithm algorithm = CompressionAlgorithm_Xpress);
```

Adds a new flag `-CompressParserStateCache` which will use the methods above to compress/decompress the parser state cache written to the `SimpleDataCacheWrapper`.

lib/Common/Common/Chakra.Common.Common.vcxproj

@@ -38,6 +38,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="$(MSBuildThisFileDirectory)CfgLogger.cpp" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)CompressionUtilities.cpp" />
     <ClCompile Include="$(MSBuildThisFileDirectory)DateUtilities.cpp" />
     <ClCompile Include="$(MSBuildThisFileDirectory)Event.cpp" />
     <ClCompile Include="$(MSBuildThisFileDirectory)Int32Math.cpp" />
@@ -59,6 +60,7 @@
     <ClInclude Include="ByteSwap.h" />
     <ClInclude Include="CommonCommonPch.h" />
     <ClInclude Include="CfgLogger.h" />
+    <ClInclude Include="CompressionUtilities.h" />
     <ClInclude Include="DateUtilities.h" />
     <ClInclude Include="Event.h" />
     <ClInclude Include="GetCurrentFrameId.h" />

lib/Common/Common/CompressionUtilities.cpp

@@ -0,0 +1,138 @@
+//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+
+#include "CommonCommonPch.h"
+#include "CompressionUtilities.h"
+
+#ifdef ENABLE_COMPRESSION_UTILITIES
+#include <compressapi.h>
+#endif
+
+#define IFFALSEGO(expr,label) do { if(!(expr)) { goto label; } } while(0);
+
+using namespace Js;
+
+DWORD ConvertCompressionAlgorithm(CompressionUtilities::CompressionAlgorithm algorithm)
+{
+    // Note: The algorithms listed in CompressionUtilities.h should be kept in-sync with those
+    // defined in compressapi.h or else we will need to do more than a simple cast here.
+    return static_cast<DWORD>(algorithm);
+}
+
+HRESULT CompressionUtilities::CompressBuffer(
+    _In_ ArenaAllocator* alloc,
+    _In_ const byte* inputBuffer,
+    _In_ size_t inputBufferByteCount,
+    _Out_ byte** compressedBuffer,
+    _Out_ size_t* compressedBufferByteCount,
+    _In_opt_ CompressionAlgorithm algorithm)
+{
+    Assert(compressedBuffer != nullptr);
+    Assert(compressedBufferByteCount != nullptr);
+
+    *compressedBuffer = nullptr;
+    *compressedBufferByteCount = 0;
+
+    HRESULT hr = E_FAIL;
+
+#ifdef ENABLE_COMPRESSION_UTILITIES
+    COMPRESSOR_HANDLE compressor = nullptr;
+    if (!CreateCompressor(ConvertCompressionAlgorithm(algorithm), nullptr, &compressor))
+    {
+        return E_FAIL;
+    }
+
+    if (algorithm == CompressionAlgorithm_Xpress || algorithm == CompressionAlgorithm_Xpress_Huff)
+    {
+        DWORD level = 0;
+        IFFALSEGO(SetCompressorInformation(compressor, COMPRESS_INFORMATION_CLASS_LEVEL, &level, sizeof(DWORD)), Error);
+    }
+
+    SIZE_T compressedByteCount = 0;
+    bool result = Compress(compressor, inputBuffer, inputBufferByteCount, nullptr, 0, &compressedByteCount);
+
+    if (!result)
+    {
+        DWORD errorCode = GetLastError();
+        if (errorCode != ERROR_INSUFFICIENT_BUFFER)
+        {
+            hr = HRESULT_FROM_WIN32(errorCode);
+            goto Error;
+        }
+    }
+
+    *compressedBuffer = AnewNoThrowArray(alloc, byte, compressedByteCount);
+    IFFALSEGO(*compressedBuffer != nullptr, Error);
+
+    SIZE_T compressedDataSize;
+    IFFALSEGO(Compress(compressor, inputBuffer, inputBufferByteCount, *compressedBuffer, compressedByteCount, &compressedDataSize), Error);
+    *compressedBufferByteCount = compressedDataSize;
+
+    hr = S_OK;
+
+Error:
+    if (compressor != nullptr)
+    {
+        CloseCompressor(compressor);
+    }
+#endif
+
+    return hr;
+}
+
+HRESULT CompressionUtilities::DecompressBuffer(
+    _In_ ArenaAllocator* alloc,
+    _In_ const byte* compressedBuffer,
+    _In_ size_t compressedBufferByteCount,
+    _Out_ byte** decompressedBuffer,
+    _Out_ size_t* decompressedBufferByteCount,
+    _In_opt_ CompressionAlgorithm algorithm)
+{
+    Assert(decompressedBuffer != nullptr);
+    Assert(decompressedBufferByteCount != nullptr);
+
+    *decompressedBuffer = nullptr;
+    *decompressedBufferByteCount = 0;
+
+    HRESULT hr = E_FAIL;
+
+#ifdef ENABLE_COMPRESSION_UTILITIES
+    DECOMPRESSOR_HANDLE decompressor = nullptr;
+    if (!CreateDecompressor(ConvertCompressionAlgorithm(algorithm), nullptr, &decompressor))
+    {
+        return E_FAIL;
+    }
+
+    SIZE_T decompressedByteCount = 0;
+    bool result = Decompress(decompressor, compressedBuffer, compressedBufferByteCount, nullptr, 0, &decompressedByteCount);
+
+    if (!result)
+    {
+        DWORD errorCode = GetLastError();
+        if (errorCode != ERROR_INSUFFICIENT_BUFFER)
+        {
+            hr = HRESULT_FROM_WIN32(errorCode);
+            goto Error;
+        }
+    }
+
+    *decompressedBuffer = AnewNoThrowArray(alloc, byte, decompressedByteCount);
+    IFFALSEGO(*decompressedBuffer != nullptr, Error);
+
+    SIZE_T uncompressedDataSize = 0;
+    IFFALSEGO(Decompress(decompressor, compressedBuffer, compressedBufferByteCount, *decompressedBuffer, decompressedByteCount, &uncompressedDataSize), Error);
+    *decompressedBufferByteCount = uncompressedDataSize;
+
+    hr = S_OK;
+
+Error:
+    if (decompressor != nullptr)
+    {
+        CloseDecompressor(decompressor);
+    }
+#endif
+
+    return hr;
+}

lib/Common/Common/CompressionUtilities.h

@@ -0,0 +1,37 @@
+//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+#pragma once
+
+namespace Js
+{
+    class CompressionUtilities
+    {
+    public:
+        enum CompressionAlgorithm : byte
+        {
+            CompressionAlgorithm_MSZip = 0x2,
+            CompressionAlgorithm_Xpress = 0x3,
+            CompressionAlgorithm_Xpress_Huff = 0x4,
+            CompressionAlgorithm_LZMS = 0x5,
+            CompressionAlgorithm_Invalid = 0xf
+        };
+
+        static HRESULT CompressBuffer(
+            _In_ ArenaAllocator* alloc,
+            _In_ const byte* inputBuffer,
+            _In_ size_t inputBufferByteCount,
+            _Out_ byte** compressedBuffer,
+            _Out_ size_t* compressedBufferByteCount,
+            _In_opt_ CompressionAlgorithm algorithm = CompressionAlgorithm_Xpress);
+
+        static HRESULT DecompressBuffer(
+            _In_ ArenaAllocator* alloc,
+            _In_ const byte* compressedBuffer,
+            _In_ size_t compressedBufferByteCount,
+            _Out_ byte** decompressedBuffer,
+            _Out_ size_t* decompressedBufferByteCount,
+            _In_opt_ CompressionAlgorithm algorithm = CompressionAlgorithm_Xpress);
+    };
+}

lib/Common/CommonDefines.h

@@ -343,6 +343,7 @@
 #define ENABLE_FOUNDATION_OBJECT
 #define ENABLE_EXPERIMENTAL_FLAGS
 #define ENABLE_WININET_PROFILE_DATA_CACHE
+#define ENABLE_COMPRESSION_UTILITIES
 #define ENABLE_BASIC_TELEMETRY
 #define ENABLE_DOM_FAST_PATH
 #define EDIT_AND_CONTINUE

lib/Common/ConfigFlagsList.h

@@ -444,6 +444,7 @@ PHASE(All)
 #define DEFAULT_CONFIG_HybridFgJitBgQueueLengthThreshold (32)
 #define DEFAULT_CONFIG_Prejit               (false)
 #define DEFAULT_CONFIG_ParserStateCache     (false)
+#define DEFAULT_CONFIG_CompressParserStateCache (false)
 #define DEFAULT_CONFIG_DeferTopLevelTillFirstCall (true)
 #define DEFAULT_CONFIG_DirectCallTelemetryStats (false)
 #define DEFAULT_CONFIG_errorStackTrace      (true)
@@ -1014,6 +1015,7 @@ FLAGNR(Phases,  DebugBreakOnPhaseBegin, "Break into debugger at the beginning of
 
 FLAGNR(Boolean, DebugWindow           , "Send console output to debugger window", false)
 FLAGNR(Boolean, ParserStateCache      , "Enable creation of parser state cache", DEFAULT_CONFIG_ParserStateCache)
+FLAGNR(Boolean, CompressParserStateCache, "Enable compression of the parser state cache", DEFAULT_CONFIG_CompressParserStateCache)
 FLAGNR(Boolean, DeferTopLevelTillFirstCall      , "Enable tracking of deferred top level functions in a script file, until the first function of the script context is parsed.", DEFAULT_CONFIG_DeferTopLevelTillFirstCall)
 FLAGNR(Number,  DeferParse            , "Minimum size of defer-parsed script (non-zero only: use /nodeferparse do disable", 0)
 FLAGNR(Boolean, DirectCallTelemetryStats, "Enables logging stats for direct call telemetry", DEFAULT_CONFIG_DirectCallTelemetryStats)

lib/Runtime/Base/ScriptContext.cpp

@@ -40,6 +40,7 @@
 #include "ByteCode/ByteCodeSerializer.h"
 #include "Language/SimpleDataCacheWrapper.h"
 #include "Core/CRC.h"
+#include "Common/CompressionUtilities.h"
 
 namespace Js
 {
@@ -2178,28 +2179,72 @@ namespace Js
         }
 
         // The block includes a 4-byte CRC before the parser state cache.
-        ULONG byteCount = blockByteCount - sizeof(uint);
+        ULONG compressedBufferByteCount = blockByteCount - sizeof(uint);
 
         // The contract for this bytecode buffer is that it is available as long as we have this ScriptContext.
         // We will use this buffer as the string table needed to back the deferred stubs as well as bytecode
         // for defer deserialized functions.
         // TODO: This, better.
         ArenaAllocator* alloc = this->SourceCodeAllocator();
-        byte* buffer = AnewArray(alloc, byte, byteCount);
+        size_t decompressedBufferByteCount = 0;
+        byte* decompressedBuffer = nullptr;
 
-        if (buffer == nullptr)
+        if (CONFIG_FLAG(CompressParserStateCache))
         {
-            return E_FAIL;
-        }
+            BEGIN_TEMP_ALLOCATOR(tempAllocator, this, _u("ByteCodeSerializer"));
+            {
+                byte* compressedBuffer = AnewNoThrowArray(tempAllocator, byte, compressedBufferByteCount);
+                if (compressedBuffer == nullptr)
+                {
+                    hr = E_FAIL;
+                    goto ExitTempAllocator;
+                }
 
-        hr = pDataCache->ReadArray(buffer, byteCount);
-        if (FAILED(hr))
+                hr = pDataCache->ReadArray(compressedBuffer, compressedBufferByteCount);
+                if (FAILED(hr))
+                {
+                    OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to read compressed parser state cache (wanted %lu bytes) (hr = 0x%08lx) for '%s'\n"), compressedBufferByteCount, hr, url);
+                    goto ExitTempAllocator;
+                }
+
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Successfully read compressed parser state cache (%lu bytes) for '%s'\n"), compressedBufferByteCount, url);
+
+                hr = Js::CompressionUtilities::DecompressBuffer(alloc, compressedBuffer, compressedBufferByteCount, &decompressedBuffer, &decompressedBufferByteCount);
+                if (FAILED(hr))
+                {
+                    OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to decompress parser state cache (hr = 0x%08lx) for '%s'\n"), hr, url);
+                    goto ExitTempAllocator;
+                }
+
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Decompressed parser state cache %lu -> %lu bytes (%.2f%%) to stream for '%s'\n"), compressedBufferByteCount, decompressedBufferByteCount, (double)compressedBufferByteCount / decompressedBufferByteCount * 100.0, url);
+            }
+ExitTempAllocator:
+            END_TEMP_ALLOCATOR(tempAllocator, this);
+
+            if (FAILED(hr))
+            {
+                goto Error;
+            }
+        }
+        else
         {
-            OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to read parser state cache (wanted %lu bytes, got %lu bytes) (hr = 0x%08lx) for '%s'\n"), byteCount, pDataCache->BytesWrittenInBlock(), hr, url);
-            return hr;
+            // We didn't compress the parser state cache so don't decompress it, just read the buffer.
+            decompressedBuffer = AnewNoThrowArray(alloc, byte, compressedBufferByteCount);
+            decompressedBufferByteCount = compressedBufferByteCount;
+            if (decompressedBuffer == nullptr)
+            {
+                return E_FAIL;
+            }
+
+            hr = pDataCache->ReadArray(decompressedBuffer, compressedBufferByteCount);
+            if (FAILED(hr))
+            {
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to read parser state cache (wanted %lu bytes) (hr = 0x%08lx) for '%s'\n"), decompressedBufferByteCount, hr, url);
+                goto Error;
+            }
         }
 
-        OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Successfully read parser state cache (%lu bytes) for '%s'\n"), byteCount, url);
+        OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Successfully read parser state cache (%lu bytes) for '%s'\n"), decompressedBufferByteCount, url);
 
         if (utf8SourceInfo != nullptr)
         {
@@ -2217,21 +2262,24 @@ namespace Js
         OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Trying to deserialize parser state cache for '%s'\n"), url);
 
         FunctionBody* functionBody = nullptr;
-        hr = Js::ByteCodeSerializer::DeserializeFromBuffer(this, grfscr, (ISourceHolder*) nullptr, srcInfo, buffer, nativeModule, &functionBody, sourceIndex);
+        hr = Js::ByteCodeSerializer::DeserializeFromBuffer(this, grfscr, (ISourceHolder*) nullptr, srcInfo, decompressedBuffer, nativeModule, &functionBody, sourceIndex);
 
         if (FAILED(hr))
         {
-            AdeleteArray(alloc, byteCount, buffer);
-
             OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to deserialize parser state cache (hr = 0x%08lx) for '%s'\n"), hr, url);
-            return hr;
+            goto Error;
         }
 
         OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Successfully deserialized parser state cache for '%s'\n"), url);
 
         *func = functionBody->GetParseableFunctionInfo();
-        *parserStateCacheBuffer = buffer;
-        *parserStateCacheByteCount = byteCount;
+        *parserStateCacheBuffer = decompressedBuffer;
+        *parserStateCacheByteCount = (DWORD)decompressedBufferByteCount;
+Error:
+        if (FAILED(hr) && decompressedBuffer != nullptr)
+        {
+            AdeleteArray(alloc, decompressedBufferByteCount, decompressedBuffer);
+        }
 #endif
 
         return hr;
@@ -2289,29 +2337,61 @@ namespace Js
 
         OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Trying to write parser state cache (%lu bytes) to stream for '%s'\n"), serializeParserStateCacheSize, url);
 
-        hr = pDataCache->StartBlock(Js::SimpleDataCacheWrapper::BlockType_ParserState, serializeParserStateCacheSize + sizeof(uint));
-
-        if (FAILED(hr))
+        BEGIN_TEMP_ALLOCATOR(tempAllocator, this, _u("ByteCodeSerializer"));
         {
-            OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write a block to the parser state cache data stream (hr = 0x%08lx) for '%s'\n"), hr, url);
-            return hr;
-        }
+            byte* compressedBuffer = nullptr;
+            size_t compressedSize = 0;
 
-        hr = pDataCache->Write(sourceCRC);
+            if (CONFIG_FLAG(CompressParserStateCache))
+            {
+                hr = Js::CompressionUtilities::CompressBuffer(tempAllocator, serializeParserStateCacheBuffer, serializeParserStateCacheSize, &compressedBuffer, &compressedSize);
 
-        OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Computed CRC value = 0x%08lx for '%s'\n"), sourceCRC, url);
+                if (FAILED(hr))
+                {
+                    OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to compress parser state cache (hr = 0x%08lx) for '%s'\n"), hr, url);
+                    goto ExitTempAllocator;
+                }
 
-        if (FAILED(hr))
-        {
-            OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write CRC data to the data stream (hr = 0x%08lx) for '%s'\n"), hr, url);
-            return hr;
-        }
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Compressed parser state cache %lu -> %lu bytes (%.2f%%) to stream for '%s'\n"), serializeParserStateCacheSize, compressedSize, (double)compressedSize / serializeParserStateCacheSize * 100.0, url);
+            }
+            else
+            {
+                // Don't compress, just pass through the parser state cache buffer
+                compressedBuffer = serializeParserStateCacheBuffer;
+                compressedSize = serializeParserStateCacheSize;
+            }
 
-        hr = pDataCache->WriteArray(serializeParserStateCacheBuffer, serializeParserStateCacheSize);
+            hr = pDataCache->StartBlock(Js::SimpleDataCacheWrapper::BlockType_ParserState, (ULONG)compressedSize + sizeof(uint));
+
+            if (FAILED(hr))
+            {
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write a block to the parser state cache data stream (hr = 0x%08lx) for '%s'\n"), hr, url);
+                goto ExitTempAllocator;
+            }
+
+            OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Computed CRC value = 0x%08lx for '%s'\n"), sourceCRC, url);
+
+            hr = pDataCache->Write(sourceCRC);
+
+            if (FAILED(hr))
+            {
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write CRC data to the data stream (hr = 0x%08lx) for '%s'\n"), hr, url);
+                goto ExitTempAllocator;
+            }
+
+            hr = pDataCache->WriteArray(compressedBuffer, (ULONG)compressedSize);
+
+            if (FAILED(hr))
+            {
+                OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write compressed parser state cache (hr = 0x%08lx) for '%s'\n"), hr, url);
+                goto ExitTempAllocator;
+            }
+        }
+ExitTempAllocator:
+        END_TEMP_ALLOCATOR(tempAllocator, this);
 
         if (FAILED(hr))
         {
-            OUTPUT_TRACE_DEBUGONLY(Js::DataCachePhase, _u(" Failed to write parser state cache (hr = 0x%08lx) for '%s'\n"), hr, url);
             return hr;
         }