Embed ICU via CMake

Add CMake external project for ICU builds. Produce a message warning
user that ICU is publised under its own term. Keep the script for
downloading ICU in case it is needed for some of Visual Studio builds,
build in a different subdirectory of `deps` (deps/thirdparty).

Refactor CommonDefines.h to move unicode include out to ChakraICU.h,
include that in one location that used to depend on unicode in
CommonDefines. Add ICU target as a dependency for components which use
ICU. Accomodate that in MSBuild files.

Add dependencies on embedded ICU for comoponents that use it (mostly via
includes).

Closes #6424

.gitignore

@@ -77,6 +77,8 @@ install_manifest.txt
 # ICU
 deps/Chakra.ICU/Chakra.ICU.props
 deps/Chakra.ICU/icu
+# CMake External Project
+deps/thirdparty
 
 # VIM
 .*.swo

CMakeLists.txt

@@ -77,6 +77,7 @@ if(ICU_SETTINGS_RESET)
     unset(NO_ICU_SH CACHE)
     unset(LOCAL_ICU_SH CACHE)
     unset(SYSTEM_ICU_SH CACHE)
+    unset(EMBED_ICU_SH CACHE)
 endif()
 
 if(CC_TARGET_OS_ANDROID_SH)
@@ -135,7 +136,47 @@ else()
     set(INTL_ICU 0)
 endif()
 
-if(ICU_INCLUDE_PATH)
+if(EMBED_ICU_SH)
+    set(EMBED_ICU 1)
+    unset(EMBED_ICU_SH CACHE)
+endif()
+
+if(EMBED_ICU AND ICU_INCLUDE_PATH)
+    message(FATAL_ERROR "Embedded ICU and ICU include path cannot be set at the same time")
+endif()
+
+if(EMBED_ICU)
+    # Keep consistent with what ICU download script used to print
+    message("Note: ICU installation and use is subject to it's publisher's licensing terms")
+
+    set(ICU_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/deps/thirdparty/icu)
+    set(ICU_DOWNLOAD_DIR ${ICU_PREFIX}/download)
+    set(ICU_SOURCE_DIR ${ICU_PREFIX}/stage)
+    set(EMBEDDED_ICU_TARGET icu4c)
+    set(ICU_INCLUDE_PATH ${ICU_PREFIX}/include)
+    set(ICU_LIBRARY_PATH ${ICU_PREFIX}/lib)
+    add_definitions(-DHAS_REAL_ICU=1)
+    add_definitions(-DHAS_ICU)
+    add_definitions(-DINTL_ICU=1)
+    set(ICU_LIBRARIES
+        ${ICU_LIBRARY_PATH}/libicuuc.a
+        ${ICU_LIBRARY_PATH}/libicui18n.a
+        ${ICU_LIBRARY_PATH}/libicudata.a
+    )
+
+    include(ExternalProject)
+    ExternalProject_Add(${EMBEDDED_ICU_TARGET}
+        PREFIX ${ICU_PREFIX}
+        DOWNLOAD_DIR ${ICU_DOWNLOAD_DIR}
+        SOURCE_DIR ${ICU_SOURCE_DIR}
+        URL https://github.com/unicode-org/icu/releases/download/release-63-2/icu4c-63_2-src.tgz
+        URL_HASH SHA512=5fa9092efd8d6da6dfc8d498e4026167fda43423eaafc754d1789cf8fd4f6e76377878ebcaa32e14f314836136b764873511a93bfbcc5419b758841cc6df8f32
+        CONFIGURE_COMMAND ${ICU_SOURCE_DIR}/source/configure --prefix=${ICU_PREFIX} --with-data-packaging=static --enable-static --disable-shared --with-library-bits=64 --disable-icuio --disable-layout --disable-tests --disable-samples
+        BUILD_COMMAND make STATICCFLAGS="-fPIC" STATICCXXFLAGS="-fPIC" STATICCPPFLAGS="-DPIC"
+        INSTALL_COMMAND make install
+        BYPRODUCTS ${ICU_LIBRARIES}
+    )
+elseif(ICU_INCLUDE_PATH)
     add_definitions(-DHAS_REAL_ICU=1)
     add_definitions(-DHAS_ICU)
     set(ICU_LIBRARY_PATH "${ICU_INCLUDE_PATH}/../lib/")

bin/ch/CMakeLists.txt

@@ -109,7 +109,6 @@ elseif(CC_TARGET_OS_OSX)
   endif()
 endif()
 
-
 target_link_libraries (ch
   ${lib_target}
   ${CC_LTO_ENABLED}

bin/ch/WScriptJsrt.cpp

@@ -3,6 +3,7 @@
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------
 #include "stdafx.h"
+#include "PlatformAgnostic/ChakraICU.h"
 #include <vector>
 
 #if defined(_X86_) || defined(_M_IX86)

bin/ch/ch.vcxproj

@@ -23,15 +23,11 @@
         %(AdditionalIncludeDirectories);
         $(MSBuildThisFileDirectory);
         $(ChakraCoreRootDirectory)Lib\Common;
+        $(ChakraCoreRootDirectory)Lib\Runtime;
         $(ChakraCoreRootDirectory)Bin\ChakraCore;
         $(IntDir);
         %(AdditionalIncludeDirectories);
       </AdditionalIncludeDirectories>
-      <!-- For ChakraICU.h -->
-      <AdditionalIncludeDirectories Condition="'$(ChakraICU)'!='false'">
-        $(ChakraCoreRootDirectory)lib\Runtime;
-        %(AdditionalIncludeDirectories)
-      </AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <ModuleDefinitionFile>ch.def</ModuleDefinitionFile>

build.sh

@@ -406,42 +406,7 @@ while [[ $# -gt 0 ]]; do
 done
 
 if [[ $USE_LOCAL_ICU == 1 ]]; then
-    LOCAL_ICU_DIR="$CHAKRACORE_DIR/deps/Chakra.ICU/icu"
-    if [[ ! -d $LOCAL_ICU_DIR ]]; then
-        "$PYTHON2_BINARY" "$CHAKRACORE_DIR/tools/icu/configure.py" 63.2 $ALWAYS_YES
-    fi
-
-    # if there is still no directory, then the user declined the license agreement
-    if [[ ! -d $LOCAL_ICU_DIR ]]; then
-        echo "You must accept the ICU license agreement in order to use this configuration"
-        exit 1
-    fi
-
-    LOCAL_ICU_DIST="$LOCAL_ICU_DIR/output"
-
-    if [ ! -d "$LOCAL_ICU_DIST" ]; then
-        set -e
-
-        pushd "$LOCAL_ICU_DIR/source"
-
-        ./configure --with-data-packaging=static\
-                    --prefix="$LOCAL_ICU_DIST"\
-                    --enable-static\
-                    --disable-shared\
-                    --with-library-bits=64\
-                    --disable-icuio\
-                    --disable-layout\
-                    --disable-tests\
-                    --disable-samples\
-                    CXXFLAGS="-fPIC"\
-                    CFLAGS="-fPIC"
-
-        ERROR_EXIT "rm -rf $LOCAL_ICU_DIST"
-        make STATICCFLAGS="-fPIC" STATICCXXFLAGS="-fPIC" STATICCPPFLAGS="-DPIC" install
-        ERROR_EXIT "rm -rf $LOCAL_ICU_DIST"
-        popd
-    fi
-    CMAKE_ICU="-DICU_INCLUDE_PATH_SH=$LOCAL_ICU_DIST/include"
+    CMAKE_ICU="-DEMBED_ICU_SH=ON"
 fi
 
 if [[ "$MAKE" == "ninja" ]]; then

lib/CMakeLists.txt

@@ -120,3 +120,23 @@ if(NOT CC_XCODE_PROJECT)
             "${CMAKE_BINARY_DIR}/include"
     )
 endif(NOT CC_XCODE_PROJECT)
+
+if (EMBED_ICU)
+  # Declare ICU dependencies in bulk.
+  # TODO Not the most idiomatic solution, need to understand if all of those
+  # libraries needed to depend on it
+  add_dependencies(Chakra.Backend ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Jsrt ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Jsrt.Core ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Parser ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Base ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.ByteCode ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Debug ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Library ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Math ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Language ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.PlatformAgnostic ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.Runtime.Types ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.SCACore ${EMBEDDED_ICU_TARGET})
+  add_dependencies(Chakra.WasmReader ${EMBEDDED_ICU_TARGET})
+endif()

lib/Common/CommonDefines.h

@@ -114,25 +114,6 @@
 #define ENABLE_UNICODE_API 1                        // Enable use of Unicode-related APIs
 #endif
 
-// Normalize ICU_VERSION for non-Kit ICU
-#if defined(HAS_ICU) && !defined(ICU_VERSION) && !defined(WINDOWS10_ICU)
-#include "unicode/uvernum.h"
-#define ICU_VERSION U_ICU_VERSION_MAJOR_NUM
-#endif
-
-// Make non-Windows Kit ICU look and act like Windows Kit ICU for better compat
-#if defined(HAS_ICU) && !defined(WINDOWS10_ICU)
-#define U_SHOW_CPLUSPLUS_API 0
-// ICU 55 (Ubuntu 16.04 system default) has uloc_toUnicodeLocale* marked as draft, which is required for Intl
-#if ICU_VERSION > 56
-#define U_DEFAULT_SHOW_DRAFT 0
-#define U_HIDE_DRAFT_API 1
-#endif
-#define U_HIDE_DEPRECATED_API 1
-#define U_HIDE_OBSOLETE_API 1
-#define U_HIDE_INTERNAL_API 1
-#endif
-
 // Language features
 #if !defined(CHAKRACORE_LITE) && (defined(_WIN32) || defined(INTL_ICU))
 #define ENABLE_INTL_OBJECT                          // Intl support

lib/Runtime/PlatformAgnostic/ChakraICU.h

@@ -15,7 +15,24 @@
 #define NTDDI_VERSION NTDDI_WIN10_RS5
 #include <icu.h>
 #pragma pop_macro("NTDDI_VERSION")
-#else
+#else // ifdef WINDOWS10_ICU
+// Normalize ICU_VERSION for non-Kit ICU
+#ifndef ICU_VERSION
+#include "unicode/uvernum.h"
+#define ICU_VERSION U_ICU_VERSION_MAJOR_NUM
+#endif
+
+// Make non-Windows Kit ICU look and act like Windows Kit ICU for better compat
+#define U_SHOW_CPLUSPLUS_API 0
+// ICU 55 (Ubuntu 16.04 system default) has uloc_toUnicodeLocale* marked as draft, which is required for Intl
+#if ICU_VERSION > 56
+#define U_DEFAULT_SHOW_DRAFT 0
+#define U_HIDE_DRAFT_API 1
+#endif
+#define U_HIDE_DEPRECATED_API 1
+#define U_HIDE_OBSOLETE_API 1
+#define U_HIDE_INTERNAL_API 1
+
 #include "unicode/ucal.h"
 #include "unicode/uclean.h"
 #include "unicode/ucol.h"
@@ -27,7 +44,7 @@
 #include "unicode/ustring.h"
 #include "unicode/unorm2.h"
 #include "unicode/upluralrules.h"
-#endif
+#endif // ifdef WINDOWS10_ICU
 
 // Different assertion code is used in ChakraFull that enforces that messages are char literals
 #ifdef _CHAKRACOREBUILD
@@ -87,4 +104,4 @@ namespace PlatformAgnostic
         }
     }
 }
-#endif
+#endif // ifdef HAS_ICU

lib/Runtime/PlatformAgnostic/Platform/CMakeLists.txt

@@ -41,6 +41,9 @@ if(NOT STATIC_LIBRARY)
   add_library (Chakra.Runtime.PlatformAgnostic.Singular STATIC
     ${PL_SOURCE_FILES}
     )
+  if (EMBED_ICU)
+    add_dependencies(Chakra.Runtime.PlatformAgnostic.Singular ${EMBEDDED_ICU_TARGET})
+  endif()
   target_include_directories (
     Chakra.Runtime.PlatformAgnostic.Singular PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
     )

pal/src/CMakeLists.txt

@@ -173,6 +173,10 @@ add_library(Chakra.Pal
   ${ARCH_SOURCES}
 )
 
+if(EMBED_ICU)
+  add_dependencies(Chakra.Pal ${EMBEDDED_ICU_TARGET})
+endif()
+
 if (NOT STATIC_LIBRARY)
   add_library(Chakra.Pal.Singular
     STATIC
@@ -196,4 +200,8 @@ if (NOT STATIC_LIBRARY)
     ${PTHREAD}
     dl
     )
+
+  if(EMBED_ICU)
+    add_dependencies(Chakra.Pal.Singular ${EMBEDDED_ICU_TARGET})
+  endif()
 endif()

pal/src/configure.cmake

@@ -913,7 +913,7 @@ else() # Anything else is Linux
   set(HAS_FTRUNCATE_LENGTH_ISSUE 0)
 endif(CMAKE_SYSTEM_NAME STREQUAL Darwin)
 
-if(NOT NO_ICU)
+if(NOT NO_ICU OR EMBED_ICU)
   if(NOT HAVE_LIBICU_UCHAR_H)
     unset(HAVE_LIBICU_UCHAR_H CACHE)
     message(FATAL_ERROR "Cannot find ICU. Try installing libicu-dev or the appropriate packages for your platform. You may also disable icu/unicode with '--no-icu' argument")

tools/icu/configure.py

@@ -4,6 +4,10 @@
 # Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 #-------------------------------------------------------------------------------------------------------
 
+#
+# TODO build.sh stopped using this script, is it needed?
+#
+
 from __future__ import print_function
 import hashlib
 import os