ChakraCore/lib/Runtime/Library/JSONScanner.cpp

//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#include "RuntimeLibraryPch.h"
#include "JSONScanner.h"

using namespace Js;

namespace JSON
{
    // -------- Scanner implementation ------------//
    JSONScanner::JSONScanner()
        : inputText(0), inputLen(0), pToken(0), stringBuffer(0), allocator(0), allocatorObject(0),
        currentRangeCharacterPairList(0), stringBufferLength(0), currentIndex(0)
    {
    }

    void JSONScanner::Finalizer()
    {
        // All dynamic memory allocated by this object is on the arena - either the one this object owns or by the
        // one shared with JSON parser - here we will deallocate ours. The others will be deallocated when JSONParser
        // goes away which should happen right after this.
        if (this->allocatorObject != nullptr)
        {
            // We created our own allocator, so we have to free it
            this->scriptContext->ReleaseTemporaryGuestAllocator(allocatorObject);
        }
    }

    void JSONScanner::Init(const char16* input, uint len, Token* pOutToken, Js::ScriptContext* sc, const char16* current, ArenaAllocator* allocator)
    {
        // Note that allocator could be nullptr from JSONParser, if we could not reuse an allocator, keep our own
        inputText = input;
        currentChar = current;
        inputLen = len;
        pToken = pOutToken;
        scriptContext = sc;
        this->allocator = allocator;
    }

    tokens JSONScanner::Scan()
    {
        pTokenString = currentChar;

        while (currentChar < inputText + inputLen)
        {
            switch(ReadNextChar())
            {
            case 0:
                //EOF
                currentChar--;
                return (pToken->tk = tkEOF);

            case '\t':
            case '\r':
            case '\n':
            case ' ':
                //WS - keep looping
                break;

            case '"':
                //check for string
                return ScanString();

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                //decimal digit starts a number
                {
                    currentChar--;

                    // we use StrToDbl() here for compat with the rest of the engine. StrToDbl() accept a larger syntax.
                    // Verify first the JSON grammar.
                    const char16* saveCurrentChar = currentChar;
                    if(!IsJSONNumber())
                    {
                       ThrowSyntaxError(JSERR_JsonBadNumber);
                    }
                    currentChar = saveCurrentChar;
                    double val;
                    const char16* end;
                    val = Js::NumberUtilities::StrToDbl(currentChar, &end, scriptContext);
                    if(currentChar == end)
                    {
                       ThrowSyntaxError(JSERR_JsonBadNumber);
                    }
                    AssertMsg(!Js::JavascriptNumber::IsNan(val), "Bad result from string to double conversion");
                    pToken->tk = tkFltCon;
                    pToken->SetDouble(val, false);
                    currentChar = end;
                    return tkFltCon;
                }

            case ',':
                return (pToken->tk = tkComma);

            case ':':
                return (pToken->tk = tkColon);

            case '[':
                return (pToken->tk = tkLBrack);

            case ']':
                return (pToken->tk = tkRBrack);

            case '-':
                return (pToken->tk = tkSub);

            case 'n':
                //check for 'null'
                if (currentChar + 2 < inputText + inputLen  && currentChar[0] == 'u' && currentChar[1] == 'l' && currentChar[2] == 'l')
                {
                    currentChar += 3;
                    return (pToken->tk = tkNULL);
                }
               ThrowSyntaxError(JSERR_JsonIllegalChar);

            case 't':
                //check for 'true'
                if (currentChar + 2 < inputText + inputLen  && currentChar[0] == 'r' && currentChar[1] == 'u' && currentChar[2] == 'e')
                {
                    currentChar += 3;
                    return (pToken->tk = tkTRUE);
                }
               ThrowSyntaxError(JSERR_JsonIllegalChar);

            case 'f':
                //check for 'false'
                if (currentChar + 3 < inputText + inputLen  && currentChar[0] == 'a' && currentChar[1] == 'l' && currentChar[2] == 's' && currentChar[3] == 'e')
                {
                    currentChar += 4;
                    return (pToken->tk = tkFALSE);
                }
               ThrowSyntaxError(JSERR_JsonIllegalChar);

            case '{':
                return (pToken->tk = tkLCurly);

            case '}':
                return (pToken->tk = tkRCurly);

            default:
               ThrowSyntaxError(JSERR_JsonIllegalChar);
            }

        }

        return (pToken->tk = tkEOF);
    }

    bool JSONScanner::IsJSONNumber()
    {
        bool firstDigitIsAZero = false;
        if (PeekNextChar() == '0')
        {
            firstDigitIsAZero = true;
            currentChar++;
        }

        //partial verification of number JSON grammar.
        while (currentChar < inputText + inputLen)
        {
            switch(ReadNextChar())
            {
            case 0:
                return false;
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                if (firstDigitIsAZero)
                {
                    return false;
                }
                break;

            case '.':
                {
                    // at least one digit after '.'
                    if(currentChar < inputText + inputLen)
                    {
                        char16 nch = ReadNextChar();
                        if('0' <= nch && nch <= '9')
                        {
                            return true;
                        }
                        else
                        {
                            return false;
                        }
                    }
                    else
                    {
                        return false;
                    }
                }
                //case 'E':
                //case 'e':
                //    return true;
            default:
                return true;
            }

            firstDigitIsAZero = false;
        }
        return true;
    }

    tokens JSONScanner::ScanString()
    {
        char16 ch;

        this->currentIndex = 0;
        this->currentString = const_cast<char16*>(currentChar);
        bool endFound = false;
        bool isStringDirectInputTextMapped = true;
        LPCWSTR bulkStart = currentChar;
        uint bulkLength = 0;

        while (currentChar < inputText + inputLen)
        {
            ch = ReadNextChar();
            int tempHex;

            if (ch == '"')
            {
                //end of the string
                endFound = true;
                break;
            }
            else if (ch <= 0x1F)
            {
                //JSON doesn't accept \u0000 - \u001f range, LS(\u2028) and PS(\u2029) are ok
               ThrowSyntaxError(JSERR_JsonIllegalChar);
            }
            else if ( 0 == ch )
            {
                currentChar--;
               ThrowSyntaxError(JSERR_JsonNoStrEnd);
            }
            else if ('\\' == ch)
            {
                //JSON escape sequence in a string \", \/, \\, \b, \f, \n, \r, \t, unicode seq
                // unlikely V5.8 regular chars are not escaped, i.e '\g'' in a string is illegal not 'g'
                if (currentChar >= inputText + inputLen )
                {
                   ThrowSyntaxError(JSERR_JsonNoStrEnd);
                }

                ch = ReadNextChar();
                switch (ch)
                {
                case 0:
                    currentChar--;
                   ThrowSyntaxError(JSERR_JsonNoStrEnd);

                case '"':
                case '/':
                case '\\':
                    //keep ch
                    break;

                case 'b':
                    ch = 0x08;
                    break;

                case 'f':
                    ch = 0x0C;
                    break;

                case 'n':
                    ch = 0x0A;
                    break;

                case 'r':
                    ch = 0x0D;
                    break;

                case 't':
                    ch = 0x09;
                    break;

                case 'u':
                    {
                        int chcode;
                        // 4 hex digits
                        if (currentChar + 3 >= inputText + inputLen)
                        {
                            //no room left for 4 hex chars
                           ThrowSyntaxError(JSERR_JsonNoStrEnd);

                        }
                        if (!Js::NumberUtilities::FHexDigit((WCHAR)ReadNextChar(), &tempHex))
                        {
                           ThrowSyntaxError(JSERR_JsonBadHexDigit);
                        }
                        chcode = tempHex * 0x1000;

                        if (!Js::NumberUtilities::FHexDigit((WCHAR)ReadNextChar(), &tempHex))
                        {
                           ThrowSyntaxError(JSERR_JsonBadHexDigit);
                        }
                        chcode += tempHex * 0x0100;

                        if (!Js::NumberUtilities::FHexDigit((WCHAR)ReadNextChar(), &tempHex))
                        {
                           ThrowSyntaxError(JSERR_JsonBadHexDigit);
                        }
                        chcode += tempHex * 0x0010;

                        if (!Js::NumberUtilities::FHexDigit((WCHAR)ReadNextChar(), &tempHex))
                        {
                           ThrowSyntaxError(JSERR_JsonBadHexDigit);
                        }
                        chcode += tempHex;
                        AssertMsg(chcode == (chcode & 0xFFFF), "Bad unicode code");
                        ch = (char16)chcode;
                    }
                    break;

                default:
                    // Any other '\o' is an error in JSON
                   ThrowSyntaxError(JSERR_JsonIllegalChar);
                }

                // flush
                this->GetCurrentRangeCharacterPairList()->Add(RangeCharacterPair((uint)(bulkStart - inputText), bulkLength, ch));

                uint oldIndex = currentIndex;
                currentIndex += bulkLength;
                currentIndex++;

                if (currentIndex < oldIndex)
                {
                    // Overflow
                    Js::Throw::OutOfMemory();
                }

                // mark the mode as 'string transformed' (no direct mapping in inputText possible)
                isStringDirectInputTextMapped = false;

                // reset (to next char)
                bulkStart = currentChar;
                bulkLength = 0;
            }
            else
            {
                // continue
                bulkLength++;
            }
        }

        if (!endFound)
        {
            // no ending '"' found
           ThrowSyntaxError(JSERR_JsonNoStrEnd);
        }

        if (isStringDirectInputTextMapped == false)
        {
            // If the last bulk is not ended with an escape character, make sure that is
            // not built into the final unescaped string
            bool shouldSkipLastCharacter = false;
            if (bulkLength > 0)
            {
                shouldSkipLastCharacter = true;
                this->GetCurrentRangeCharacterPairList()->Add(RangeCharacterPair((uint)(bulkStart - inputText), bulkLength, _u('\0')));
                uint oldIndex = currentIndex;
                currentIndex += bulkLength;
                if (currentIndex < oldIndex)
                {
                    // Overflow
                    Js::Throw::OutOfMemory();
                }
            }

            this->BuildUnescapedString(shouldSkipLastCharacter);
            this->GetCurrentRangeCharacterPairList()->Clear();
            this->currentString = this->stringBuffer;
        }
        else
        {
            // make currentIndex the length (w/o the \0)
            currentIndex = bulkLength;

            OUTPUT_TRACE_DEBUGONLY(Js::JSONPhase, _u("ScanString(): direct-mapped string as '%.*s'\n"),
                GetCurrentStringLen(), GetCurrentString());
        }

        return (pToken->tk = tkStrCon);
    }

    void JSONScanner::BuildUnescapedString(bool shouldSkipLastCharacter)
    {
        AssertMsg(this->allocator != nullptr, "We must have built the allocator");
        AssertMsg(this->currentRangeCharacterPairList != nullptr, "We must have built the currentRangeCharacterPairList");
        AssertMsg(this->currentRangeCharacterPairList->Count() > 0, "We need to build the current string only because we have escaped characters");

        // Step 1: Ensure the buffer has sufficient space
        int requiredSize = this->GetCurrentStringLen();
        if (requiredSize > this->stringBufferLength)
        {
            if (this->stringBuffer)
            {
                AdeleteArray(this->allocator, this->stringBufferLength, this->stringBuffer);
                this->stringBuffer = nullptr;
            }

            this->stringBuffer = AnewArray(this->allocator, char16, requiredSize);
            this->stringBufferLength = requiredSize;
        }

        // Step 2: Copy the data to the buffer
        int totalCopied = 0;
        char16* begin_copy = this->stringBuffer;
        int lastCharacterIndex = this->currentRangeCharacterPairList->Count() - 1;
        for (int i = 0; i <= lastCharacterIndex; i++)
        {
            RangeCharacterPair data = this->currentRangeCharacterPairList->Item(i);
            int charactersToCopy = data.m_rangeLength;
            js_wmemcpy_s(begin_copy, charactersToCopy, this->inputText + data.m_rangeStart, charactersToCopy);
            begin_copy += charactersToCopy;
            totalCopied += charactersToCopy;

            if (i == lastCharacterIndex && shouldSkipLastCharacter)
            {
                continue;
            }

            *begin_copy = data.m_char;
            begin_copy++;
            totalCopied++;
        }

        if (totalCopied != requiredSize)
        {
            OUTPUT_TRACE_DEBUGONLY(Js::JSONPhase, _u("BuildUnescapedString(): allocated size = %d != copying size %d\n"), requiredSize, totalCopied);
            AssertMsg(totalCopied == requiredSize, "BuildUnescapedString(): The allocated size and copying size should match.");
        }

        OUTPUT_TRACE_DEBUGONLY(Js::JSONPhase, _u("BuildUnescapedString(): unescaped string as '%.*s'\n"), GetCurrentStringLen(), this->stringBuffer);
    }

    JSONScanner::RangeCharacterPairList* JSONScanner::GetCurrentRangeCharacterPairList(void)
    {
        if (this->currentRangeCharacterPairList == nullptr)
        {
            if (this->allocator == nullptr)
            {
                this->allocatorObject = this->scriptContext->GetTemporaryGuestAllocator(_u("JSONScanner"));
                this->allocator = this->allocatorObject->GetAllocator();
            }

            this->currentRangeCharacterPairList = Anew(this->allocator, RangeCharacterPairList, this->allocator, 4);
        }

        return this->currentRangeCharacterPairList;
    }
} // namespace JSON