[MERGE #3134 @dilijev] Remove BOM from C++ source files; sanitize non-ASCII files in sources and tools.

Merge pull request #3134 from dilijev:cleanup-bom-ascii

build.sh

@@ -326,7 +326,7 @@ if [[ $SHOULD_EMBED_ICU == 1 ]]; then
         echo -e "\nThis script will download ICU-LIB from\n${ICU_URL}\n"
         echo "It is licensed to you by its publisher, not Microsoft."
         echo "Microsoft is not responsible for the software."
-        echo "Your installation and use of ICU-LIB is subject to the publisher’s terms available here:"
+        echo "Your installation and use of ICU-LIB is subject to the publisher's terms available here:"
         echo -e "http://www.unicode.org/copyright.html#License\n"
         echo -e "----------------------------------------------------------------\n"
         echo "If you don't agree, press Ctrl+C to terminate"

jenkins/check_file_eol.sh

@@ -20,7 +20,7 @@ fi
 # We can't rely on dos2unix being installed, so simply grep for the CR octet 0x0d via xxd.
 # We don't want to simply detect a literal 0d in the file or output so configure xxd to emit
 # octets in such a way that we can grep for the CR octet and not accidentally detect
-# text of the file or 0d spanning 2 octets in xxd output, e.g., 20d1 (' Ñ').
+# text of the file or 0d spanning 2 octets in xxd output (e.g. the 0d in 20d1)
 xxd -i -c 16 $1 | grep '0x0d' > $ERRFILETEMP
 if [ $? -eq 0 ]; then # grep found matches ($?==0), so we found CR (0x0d) in the file
     echo "ERROR: CR (0x0d) was introduced in $1" >> $ERRFILE

lib/Backend/GlobOptBailOut.cpp

@@ -986,7 +986,7 @@ GlobOpt::FillBailOutInfo(BasicBlock *block, BailOutInfo * bailOutInfo)
 #ifdef _M_IX86
                 if (this->currentRegion && (this->currentRegion->GetType() == RegionTypeTry || this->currentRegion->GetType() == RegionTypeFinally))
                 {
-                    // For a bailout in argument evaluation from an EH region, the esp is offset by the TryCatch helper�s frame. So, the argouts are not actually pushed at the
+                    // For a bailout in argument evaluation from an EH region, the esp is offset by the TryCatch helper's frame. So, the argouts are not actually pushed at the
                     // offsets stored in the bailout record, which are relative to ebp. Need to restore the argouts from the actual value of esp before calling the Bailout helper.
                     // For nested calls, argouts for the outer call need to be restored from an offset of stack-adjustment-done-by-the-inner-call from esp.
                     if (startCallNumber + 1 == bailOutInfo->startCallCount)

lib/Common/DataStructures/BigInt.cpp

@@ -1,4 +1,4 @@
-//-------------------------------------------------------------------------------------------------------
+//-------------------------------------------------------------------------------------------------------
 // Copyright (C) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------

lib/Common/DataStructures/BigInt.h

@@ -1,4 +1,4 @@
-//-------------------------------------------------------------------------------------------------------
+//-------------------------------------------------------------------------------------------------------
 // Copyright (C) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------

lib/Runtime/Library/MathLibrary.cpp

@@ -1,4 +1,4 @@
-//-------------------------------------------------------------------------------------------------------
+//-------------------------------------------------------------------------------------------------------
 // Copyright (C) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------
@@ -1520,9 +1520,9 @@ namespace Js
 
         // ES6 20.2.2.18 Math.hypot(value1, value2, ...values)
         // If no arguments are passed, the result is +0.
-        // If any argument is +∞, the result is +∞.
-        // If any argument is -∞, the result is +∞.
-        // If no argument is +∞ or -∞, and any argument is NaN, the result is NaN.
+        // If any argument is +Infinity, the result is +Infinity.
+        // If any argument is -Infinity, the result is +Infinity.
+        // If no argument is +Infinity or -Infinity, and any argument is NaN, the result is NaN.
         // If all arguments are either +0 or -0, the result is +0.
 
         double result = JavascriptNumber::k_Zero; // If there are no arguments return value is positive zero.
@@ -1591,7 +1591,7 @@ namespace Js
             {
                 if (JavascriptNumber::IsNan(doubleVal))
                 {
-                    //Even though we found NaN, we still need to validate none of the other arguments are +∞ or -∞
+                    //Even though we found NaN, we still need to validate none of the other arguments are +Infinity or -Infinity
                     foundNaN = true;
                 }
                 else

pal/src/cruntime/printf.cpp

@@ -1186,23 +1186,23 @@ int PAL_vsscanf(LPCSTR Buffer, LPCSTR Format, va_list ap)
 
                 if (Store)
                 {
-                    // sscanf_s requires that if we are trying to read "%s" or "%c" or “%[“, then
+                    // sscanf_s requires that if we are trying to read "%s" or "%c" or "%[", then
                     // the size of the buffer must follow the buffer we are trying to read into.
                     voidPtr = va_arg(ap, LPVOID);
                     unsigned typeLen = 0;
                     if ((Type == SCANF_TYPE_STRING) || (Type == SCANF_TYPE_BRACKETS))
                     {
-                        // Since this is not a Safe CRT API we don’t really know the size of the destination
+                        // Since this is not a Safe CRT API we don't really know the size of the destination
                         // buffer provided by the caller. So we have to assume that the caller has allocated
                         // enough space to hold either the width specified in the format or the entire input
-                        // string plus ‘\0’.
+                        // string plus '\0'.
                         typeLen = ((Width > 0) ? Width : strlen(Buffer)) + 1;
                     }
                     else if (Type == SCANF_TYPE_CHAR)
                     {
                         // Check whether the format string contains number of characters
                         // that should be read from the input string.
-                        // Note: ‘\0’ does not get appended in the “%c” case.
+                        // Note: '\0' does not get appended in the "%c" case.
                         typeLen = (Width > 0) ? Width : 1;
                     }
 
@@ -1431,17 +1431,17 @@ int PAL_wvsscanf(LPCWSTR Buffer, LPCWSTR Format, va_list ap)
                         unsigned typeLen = 0;
                         if (Type == SCANF_TYPE_STRING)
                         {
-                            // We don’t really know the size of the destination buffer provided by the
+                            // We don't really know the size of the destination buffer provided by the
                             // caller. So we have to assume that the caller has allocated enough space
                             // to hold either the width specified in the format or the entire input
-                            // string plus ‘\0’.
+                            // string plus '\0'.
                             typeLen = ((Width > 0) ? Width : PAL_wcslen(Buffer)) + 1;
                         }
                         else if (Type == SCANF_TYPE_CHAR)
                         {
                             // Check whether the format string contains number of characters
                             // that should be read from the input string.
-                            // Note: ‘\0’ does not get appended in the “%c” case.
+                            // Note: '\0' does not get appended in the "%c" case.
                             typeLen = (Width > 0) ? Width : 1;
                         }
 

tools/compile_clang.sh

@@ -22,7 +22,7 @@ WARN_LICENSE () {
     echo -e "This script will download LLVM/CLANG and LLVM Gold Bintools from\n${CC_URL}\n"
     echo "These software are licensed to you by its publisher(s), not Microsoft."
     echo "Microsoft is not responsible for the software."
-    echo "Your installation and use of the software is subject to the publisher’s terms available here:"
+    echo "Your installation and use of the software is subject to the publisher's terms available here:"
     echo -e "http://llvm.org/docs/DeveloperPolicy.html#license\nhttp://llvm.org/docs/GoldPlugin.html#licensing"
     echo -e "${ERROR_COLOR}"
     echo -e "----------------------------------------------------------------\n"
@@ -105,7 +105,7 @@ if [ ! -d ./cc-toolchain/src/llvm/projects/compiler-rt ]; then
     cd "${ROOT}/src/"
 
     echo "Downloading LLVM ${LLVM_VERSION}"
-    wget –quiet "http://llvm.org/releases/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
+    wget --quiet "http://llvm.org/releases/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
     tar -xf "llvm-${LLVM_VERSION}.src.tar.xz"
     if [ $? == 0 ]; then
         rm "llvm-${LLVM_VERSION}.src.tar.xz"
@@ -116,7 +116,7 @@ if [ ! -d ./cc-toolchain/src/llvm/projects/compiler-rt ]; then
 
     cd llvm/tools/
     echo "Downloading Clang ${LLVM_VERSION}"
-    wget –quiet "http://llvm.org/releases/${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
+    wget --quiet "http://llvm.org/releases/${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
     tar -xf "cfe-${LLVM_VERSION}.src.tar.xz"
     if [ $? == 0 ]; then
         mv "cfe-${LLVM_VERSION}.src" clang
@@ -128,7 +128,7 @@ if [ ! -d ./cc-toolchain/src/llvm/projects/compiler-rt ]; then
     mkdir -p ../projects/
     cd ../projects/
     echo "Downloading Compiler-RT ${LLVM_VERSION}"
-    wget –quiet "http://llvm.org/releases/${LLVM_VERSION}/compiler-rt-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
+    wget --quiet "http://llvm.org/releases/${LLVM_VERSION}/compiler-rt-${LLVM_VERSION}.src.tar.xz" >/dev/null 2>&1
     tar -xf "compiler-rt-${LLVM_VERSION}.src.tar.xz"
     if [ $? == 0 ]; then
         mv "compiler-rt-${LLVM_VERSION}.src" compiler-rt