::------------------------------------------------------------------------------------------------------- :: Copyright (C) Microsoft. All rights reserved. :: Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. ::------------------------------------------------------------------------------------------------------- :: ============================================================================ :: :: runtests.cmd :: :: Runs checkin tests using the ch.exe on the path, in 2 variants: :: :: -maxInterpretCount:1 -maxSimpleJitRunCount:1 -bgjit- :: :: :: Logs are placed into: :: :: logs\interpreted :: logs\dynapogo :: :: User specified variants: :: logs\forcedeferparse :: logs\nodeferparse :: logs\forceundodefer :: logs\bytecodeserialized (serialized to byte codes) :: logs\forceserialized (force bytecode serialization internally) :: :: ============================================================================ @echo off setlocal goto :main :: ============================================================================ :: Print usage :: ============================================================================ :printUsage echo runtests.cmd -x86^|-x64^|-arm -debug^|-test [options] echo. echo Required switches: echo. echo Specify architecture of ChakraCore: echo. echo -x86 Build arch of binaries is x86 echo -x64 Build arch of binaries is x64 echo -arm Build arch of binaries is ARM echo -arm64 Build arch of binaries is ARM64 echo. echo Specify type of ChakraCore: echo. echo -debug Build type of binaries is debug echo -test Build type of binaries is test echo -codecoverage Build type of binaries is codecoverage echo. echo Shorthand combinations can be used, e.g. -x64debug echo. echo Note: release build type currently unsupported by ch.exe echo. echo Options: echo. echo -dirs dirname Run only the specified directory echo -rebase Create .rebase file on baseline comparision failure echo -stoponerror Stop testing after first failure (will finish current execution) :: TODO Add more usage help goto :eof :: ============================================================================ :: Print how to get help :: ============================================================================ :printGetHelp echo For help use runtests.cmd -? goto :eof :: ============================================================================ :: Main script :: ============================================================================ :main if not exist %cd%\rlexedirs.xml ( echo Error: rlexedirs.xml not found in current directory. echo runtests.cmd must be run from a test root directory containing rlexedirs.xml. exit /b 1 ) call :initVars call :parseArgs %* if not "%fShowUsage%" == "" ( call :printUsage goto :eof ) call :validateArgs if not "%fShowGetHelp%" == "" ( call :printGetHelp goto :eof ) call :configureVars set _logsRoot=%cd%\logs call :doSilent del /s /q profile.dpl.* for %%i in (%_Variants%) do ( set _TESTCONFIG=%%i call :RunOneVariant ) call :cleanUp for %%i in (%_Variants%) do ( echo. echo ######## Logs for %%i variant ######## if exist %_logsRoot%\%_BuildArch%_%_BuildType%\%%i\rl.log ( type %_logsRoot%\%_BuildArch%_%_BuildType%\%%i\rl.log ) else ( echo ERROR: Log file '%_logsRoot%\%_BuildArch%_%_BuildType%\%%i\rl.log' does not exist ) ) if "%_HadFailures%" NEQ "0" ( echo -- runtests.cmd ^>^> Tests failed. See logs for details. ) else ( echo -- runtests.cmd ^>^> All tests passed! ) echo -- runtests.cmd ^>^> exiting with exit code %_HadFailures% exit /b %_HadFailures% :: ============================================================================ :: Parse the user arguments into environment variables :: ============================================================================ :parseArgs :NextArgument if "%1" == "-?" set fShowUsage=1& goto :ArgOk if "%1" == "/?" set fShowUsage=1& goto :ArgOk if /i "%1" == "-x86" set _BuildArch=x86& goto :ArgOk if /i "%1" == "-x64" set _BuildArch=x64& goto :ArgOk if /i "%1" == "-arm" set _BuildArch=arm& goto :ArgOk if /i "%1" == "-arm64" set _BuildArch=arm64& goto :ArgOk if /i "%1" == "-debug" set _BuildType=debug& goto :ArgOk if /i "%1" == "-test" set _BuildType=test& goto :ArgOk if /i "%1" == "-codecoverage" set _BuildType=codecoverage& goto :ArgOk if /i "%1" == "-x86debug" set _BuildArch=x86&set _BuildType=debug& goto :ArgOk if /i "%1" == "-x64debug" set _BuildArch=x64&set _BuildType=debug& goto :ArgOk if /i "%1" == "-armdebug" set _BuildArch=arm&set _BuildType=debug& goto :ArgOk if /i "%1" == "-arm64debug" set _BuildArch=arm64&set _BuildType=debug& goto :ArgOk if /i "%1" == "-x86test" set _BuildArch=x86&set _BuildType=test& goto :ArgOk if /i "%1" == "-x64test" set _BuildArch=x64&set _BuildType=test& goto :ArgOk if /i "%1" == "-armtest" set _BuildArch=arm&set _BuildType=test& goto :ArgOk if /i "%1" == "-arm64test" set _BuildArch=arm64&set _BuildType=test& goto :ArgOk if /i "%1" == "-x86codecoverage" set _BuildArch=x86&set _BuildType=codecoverage& goto :ArgOk if /i "%1" == "-x64codecoverage" set _BuildArch=x64&set _BuildType=codecoverage& goto :ArgOk if /i "%1" == "-armcodecoverage" set _BuildArch=arm&set _BuildType=codecoverage& goto :ArgOk if /i "%1" == "-arm64codecoverage" set _BuildArch=arm64&set _BuildType=codecoverage& goto :ArgOk if /i "%1" == "-binary" set _Binary=-binary:%2& goto :ArgOkShift2 if /i "%1" == "-bindir" set _BinDir=%~f2& goto :ArgOkShift2 if /i "%1" == "-dirs" set _DIRS=-dirs:%~2& goto :ArgOkShift2 if /i "%1" == "-win7" set TARGET_OS=win7& goto :ArgOk if /i "%1" == "-win8" set TARGET_OS=win8& goto :ArgOk if /i "%1" == "-winBlue" set TARGET_OS=winBlue& goto :ArgOk if /i "%1" == "-win10" set TARGET_OS=win10& goto :ArgOk if /i "%1" == "-nottags" set _NOTTAGS=%_NOTTAGS% -nottags:%~2& goto :ArgOkShift2 if /i "%1" == "-tags" set _TAGS=%_TAGS% -tags:%~2& goto :ArgOkShift2 if /i "%1" == "-dirtags" set _DIRTAGS=%_DIRTAGS% -dirtags:%~2& goto :ArgOkShift2 if /i "%1" == "-dirnottags" set _DIRNOTTAGS=%_DIRNOTTAGS% -dirnottags:%~2& goto :ArgOkShift2 if /i "%1" == "-includeSlow" set _includeSlow=1& goto :ArgOk if /i "%1" == "-onlySlow" set _onlySlow=1& goto :ArgOk if /i "%1" == "-quiet" set _quiet=-quiet& goto :ArgOk :: TODO Consider removing -drt and exclude_drt in some reasonable manner if /i "%1" == "-drt" set _drt=1& set _NOTTAGS=%_NOTTAGS% -nottags:exclude_drt& goto :ArgOk if /i "%1" == "-rebase" set _rebase=-rebase& goto :ArgOk if /i "%1" == "-stoponerror" set _stoponerror=-stoponerror& goto :ArgOk if /i "%1" == "-rundebug" set _RUNDEBUG=1& goto :ArgOk :: TODO Figure out best way to specify build arch for tests that are excluded to specific archs if /i "%1" == "-platform" set _buildArch=%2& goto :ArgOkShift2 :: TODO Figure out best way to specify build type for tests that are excluded to specific type (chk, fre, etc) if /i "%1" == "-buildType" set _buildType=%2& goto :ArgOkShift2 if /i "%1" == "-binaryRoot" set _binaryRoot=%~f2& goto :ArgOkShift2 if /i "%1" == "-variants" set _Variants=%~2& goto :ArgOkShift2 if /i "%1" == "-cleanupall" set _CleanUpAll=1& goto :ArgOk if /i "%1" == "-noprogramoutput" set _NoProgramOutput=-noprogramoutput& goto :ArgOk if /i "%1" == "-onlyassertoutput" set _OnlyAssertOutput=-onlyassertoutput& goto :ArgOk ::Extra ch.exe command line flags if /i "%1" == "-ExtraHostFlags" set _ExtraHostFlags=%~2& goto :ArgOkShift2 if /i "%1" == "-DumpOnCrash" set _DumpOnCrash=1& goto :ArgOk if /i "%1" == "-CrashOnException" set _CrashOnException=1& goto :ArgOk ::Timeout flags if /i "%1" == "-timeout" set _TestTimeout=%~2& goto :ArgOkShift2 if /i "%1" == "-timeoutRetries" set _TestTimeoutRetries=%~2& goto :ArgOkShift2 if /i "%1" == "-extraVariants" ( :: Extra variants are specified by the user but not run by default. if "%_ExtraVariants%" == "" ( set _ExtraVariants=%~2 ) else ( set _ExtraVariants=%_ExtraVariants%,%~2 ) goto :ArgOkShift2 ) :: Defined here are shorthand versions for specifying :: extra variants when running. if /i "%1" == "-parser" ( if "%_ExtraVariants%" == "" ( set _ExtraVariants=forcedeferparse,nodeferparse,forceundodefer ) else ( set _ExtraVariants=%_ExtraVariants%,forcedeferparse,nodeferparse,forceundodefer ) goto :ArgOk ) if /i "%1" == "-serialization" ( if "%_ExtraVariants%" == "" ( set _ExtraVariants=bytecodeserialized,forceserialized ) else ( set _ExtraVariants=%_ExtraVariants%,bytecodeserialized,forceserialized ) goto :ArgOk ) if /i "%1" == "-disablejit" ( set _Variants=disable_jit goto :ArgOk ) if /i "%1" == "-lite" ( set _Variants=lite goto :ArgOk ) if /i "%1" == "-nightly" ( set _nightly=1 if "%_ExtraVariants%" == "" ( set _ExtraVariants=mediumlayout,largelayout,forceserialized ) else ( set _ExtraVariants=%_ExtraVariants%,mediumlayout,largelayout,forceserialized ) goto :ArgOk ) if not "%1" == "" echo Unknown argument: %1 & set fShowGetHelp=1 goto :eof :ArgOkShift2 shift :ArgOk shift goto :NextArgument :: ============================================================================ :: Initialize batch script variables to defaults :: ============================================================================ :initVars set _HadFailures=0 set _RootDir=%~dp0.. set _BinDir=%_RootDir%\Build\VcBuild\Bin set _BuildArch= set _BuildType= set _Binary=-binary:ch.exe set _Variants= set _TAGS= set _NOTTAGS= set _DIRNOTTAGS= set _DIRTAGS= set _drt= set _rebase= set _stoponerror= set _ExtraVariants= set _dynamicprofilecache=-dynamicprofilecache:profile.dpl set _dynamicprofileinput=-dynamicprofileinput:profile.dpl set _includeSlow= set _onlySlow= set _CleanUpAll= set _nightly= set TARGET_OS=win10 set _quiet= set _ExtraHostFlags= set _DumpOnCrash= set _CrashOnException= set _NoProgramOutput= set _OnlyAssertOutput= goto :eof :: ============================================================================ :: Validate that required arguments were specified :: ============================================================================ :validateArgs if "%_BuildArch%" == "" ( echo Error missing required build architecture or build type switch set fShowGetHelp=1 goto :eof ) if "%_BuildType%" == "" ( echo Error missing required build architecture or build type switch set fShowGetHelp=1 ) if not exist %_binDir%\%_BuildArch%_%_BuildType%\%_Binary:~8% ( echo Error missing binary %_binDir%\%_BuildArch%_%_BuildType%\%_Binary:~8% set fShowGetHelp=1 ) goto :eof :: ============================================================================ :: Configure the script variables and environment based on parsed arguments :: ============================================================================ :configureVars echo Adding to PATH: %_binDir%\%_BuildArch%_%_BuildType% set path=%_binDir%\%_BuildArch%_%_BuildType%;%path% :: If the user didn't specify explicit variants then do the defaults if "%_Variants%"=="" set _Variants=interpreted,dynapogo :: If the user specified extra variants to run (i.e. in addition to the defaults), include them. if not "%_ExtraVariants%" == "" set _Variants=%_Variants%,%_ExtraVariants% if not "%_nightly%" == "1" ( set _NOTTAGS=%_NOTTAGS% -nottags:nightly ) else ( set _NOTTAGS=%_NOTTAGS% -nottags:exclude_nightly ) if "%_includeSlow%%_onlySlow%" == "" ( set _NOTTAGS=%_NOTTAGS% -nottags:Slow ) if "%_onlySlow%" == "1" ( set _TAGS=%_TAGS% -tags:Slow ) if not "%NUM_RL_THREADS%" == "" ( set _RL_THREAD_FLAGS=-threads:%NUM_RL_THREADS% ) if "%_DIRS%" == "" ( set _DIRS=-all ) set _NOTTAGS=%_NOTTAGS% -nottags:exclude_windows set _BuildArchMapped=%_BuildArch% set _BuildTypeMapped=%_BuildType% :: _NewBuildArchMapped and _NewBuildTypeMapped store new terms of the build and architecture :: _NewBuildArchMapped: x86/x64 _BuildArchMapped:x86/amd64 :: _NewBuildTypeMapped: debug/test _BuildTypeMapped:chk/fre set _NewBuildArchMapped=%_BuildArch% set _NewBuildTypeMapped=%_BuildType% :: Map new build arch and type names to old names until rl test tags are :: updated to the new names if "%_BuildArchMapped%" == "x64" set _BuildArchMapped=amd64 if "%_BuildTypeMapped%" == "debug" set _BuildTypeMapped=chk if "%_BuildTypeMapped%" == "test" set _BuildTypeMapped=fre if "%_BuildTypeMapped%" == "codecoverage" ( set _BuildTypeMapped=fre set _NewBuildTypeMapped=test ) if "%_BuildArch%" == "arm" set _NOTTAGS=%_NOTTAGS% -nottags:require_asmjs -nottags:require_wasm if "%_BuildArch%" == "arm64" set _NOTTAGS=%_NOTTAGS% -nottags:require_asmjs -nottags:require_wasm if "%Disable_JIT%" == "1" ( set _dynamicprofilecache= set _dynamicprofileinput= ) goto :eof :: ============================================================================ :: Run one variant :: ============================================================================ :RunOneVariant if exist %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ( rd /q /s %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ) if %_HadFailures% NEQ 0 ( if "%_stoponerror%" NEQ "" ( goto :eof ) ) if "%_BuildType%" == "test" ( rem bytecode layout switches not available in test build if "%_TESTCONFIG%"=="largelayout" ( if not exist %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ( mkdir %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ) echo. > %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG%\rl.log goto :eof ) if "%_TESTCONFIG%"=="mediumlayout" ( if not exist %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ( mkdir %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% ) echo. > %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG%\rl.log goto :eof ) ) set _OLD_CC_FLAGS=%EXTRA_CC_FLAGS% set EXTRA_RL_FLAGS=-appendtestnametoextraccflags set _exclude_serialized= if "%_BuildType%" == "debug" ( rem Enabling storing dumps on user directory. set _DumpOnCrash=1 ) set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% %_ExtraHostFlags% if not "%_DumpOnCrash%" == "" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -DumpOnCrash ) if not "%_CrashOnException%" == "" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -CrashOnException ) if "%_Binary%" == "-binary:ch.exe" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -WERExceptionSupport -ExtendedErrorStackForTestHost -BaselineMode ) if "%_TESTCONFIG%"=="interpreted" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -maxInterpretCount:1 -maxSimpleJitRunCount:1 -bgjit- %_dynamicprofilecache% ) if "%_TESTCONFIG%"=="nonative" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -nonative set EXTRA_RL_FLAGS=-nottags:exclude_interpreted -nottags:fails_interpreted ) :: DisableJit is different from NoNative in that NoNative can be used :: with builds which include backend code, whereas DisableJit doesn't have :: backend code linked in, and also disables other features that incidentally :: depends on the backend like dynamic profile, asmjs, simdjs, background parsing etc. :: TODO: Re-enable interpreter mode asmjs and simdjs if "%_TESTCONFIG%"=="disable_jit" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -nonative set EXTRA_RL_FLAGS=-nottags:exclude_interpreted -nottags:fails_interpreted -nottags:require_backend ) else ( set EXTRA_RL_FLAGS=%EXTRA_RL_FLAGS% -nottags:require_disable_jit ) if "%_TESTCONFIG%"=="lite" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -nonative set EXTRA_RL_FLAGS=-nottags:exclude_interpreted -nottags:fails_interpreted -nottags:require_backend -nottags:require_debugger -nottags:Intl ) if "%_TESTCONFIG%"=="dynapogo" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -forceNative -off:simpleJit -bgJitDelay:0 %_dynamicprofileinput% ) :: Variants after here are user supplied variants (not run by default). if "%_TESTCONFIG%"=="forcedeferparse" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -forceDeferParse %_dynamicprofilecache% set _exclude_forcedeferparse=-nottags:exclude_forcedeferparse ) if "%_TESTCONFIG%"=="nodeferparse" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -noDeferParse %_dynamicprofilecache% set _exclude_nodeferparse=-nottags:exclude_nodeferparse ) if "%_TESTCONFIG%"=="forceundodefer" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -forceUndoDefer %_dynamicprofilecache% set _exclude_forceundodefer=-nottags:exclude_forceundodefer ) if "%_TESTCONFIG%"=="bytecodeserialized" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -recreatebytecodefile -serialized:%TEMP%\ByteCode set _exclude_serialized=-nottags:exclude_serialized ) if "%_TESTCONFIG%"=="forceserialized" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -forceserialized set EXTRA_RL_FLAGS=-nottags:exclude_forceserialized set _exclude_serialized=-nottags:exclude_serialized ) if "%_TESTCONFIG%"=="mediumlayout" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -MediumByteCodeLayout -forceserialized set EXTRA_RL_FLAGS=-nottags:exclude_bytecodelayout -nottags:exclude_forceserialized set _exclude_serialized=-nottags:exclude_serialized ) if "%_TESTCONFIG%"=="largelayout" ( set EXTRA_CC_FLAGS=%EXTRA_CC_FLAGS% -LargeByteCodeLayout -forceserialized set EXTRA_RL_FLAGS=-nottags:exclude_bytecodelayout -nottags:exclude_forceserialized set _exclude_serialized=-nottags:exclude_serialized ) if not "%_TestTimeout%" == "" ( set EXTRA_RL_FLAGS=%EXTRA_RL_FLAGS% -timeout:%_TestTimeout% ) if not "%_TestTimeoutRetries%" == "" ( set EXTRA_RL_FLAGS=%EXTRA_RL_FLAGS% -timeoutRetries:%_TestTimeoutRetries% ) echo. echo ############# Starting %_TESTCONFIG% variant ############# call :doSilent del /q %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG%\rl* call :doSilent md %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% set _rlArgs=%_Binary% set _rlArgs=%_rlArgs% -target:%_BuildArchMapped% set _rlArgs=%_rlArgs% -nottags:fail set _rlArgs=%_rlArgs% %_RL_THREAD_FLAGS% set _rlArgs=%_rlArgs% %_DIRS% set _rlArgs=%_rlArgs% -verbose set _rlArgs=%_rlArgs% %_TAGS% set _rlArgs=%_rlArgs% %_NOTTAGS% set _rlArgs=%_rlArgs% %_DIRTAGS% set _rlArgs=%_rlArgs% %_DIRNOTTAGS% set _rlArgs=%_rlArgs% -nottags:fails_%_TESTCONFIG% set _rlArgs=%_rlArgs% -nottags:exclude_%_TESTCONFIG% set _rlArgs=%_rlArgs% -nottags:exclude_%TARGET_OS% set _rlArgs=%_rlArgs% -nottags:exclude_%_BuildArchMapped% set _rlArgs=%_rlArgs% -nottags:exclude_%_NewBuildArchMapped% set _rlArgs=%_rlArgs% -nottags:exclude_%_BuildTypeMapped% set _rlArgs=%_rlArgs% -nottags:exclude_%_NewBuildTypeMapped% set _rlArgs=%_rlArgs% %_exclude_serialized% set _rlArgs=%_rlArgs% %_exclude_forcedeferparse% set _rlArgs=%_rlArgs% %_exclude_nodeferparse% set _rlArgs=%_rlArgs% %_exclude_forceundodefer% set _rlArgs=%_rlArgs% %_NoProgramOutput% set _rlArgs=%_rlArgs% %_OnlyAssertOutput% set _rlArgs=%_rlArgs% %_quiet% set _rlArgs=%_rlArgs% -exe set _rlArgs=%_rlArgs% %EXTRA_RL_FLAGS% set _rlArgs=%_rlArgs% %_rebase% set _rlArgs=%_rlArgs% %_stoponerror% set REGRESS=%CD% call :do rl %_rlArgs% if %ERRORLEVEL% NEQ 0 set _HadFailures=1 call :do move /Y %_logsRoot%\*.log %_logsRoot%\%_BuildArch%_%_BuildType%\%_TESTCONFIG% if %ERRORLEVEL% NEQ 0 set _HadFailures=1 set EXTRA_CC_FLAGS=%_OLD_CC_FLAGS% goto :eof :: ============================================================================ :: Clean up left over files :: ============================================================================ :cleanUp call :doSilent del /s *.bc call :doSilent del /s *.out call :doSilent del /s *.dpl call :doSilent del /s profile.dpl.* call :doSilent del /s testout* if "%_CleanUpAll%" == "1" ( call :doSilent del /s *.rebase ) goto :eof :: ============================================================================ :: Echo a command line before executing it :: ============================================================================ :do echo ^>^> %* cmd /s /c "%*" if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL% goto :eof :: ============================================================================ :: Echo a command line before executing it and redirect the command's output :: to nul :: ============================================================================ :doSilent echo ^>^> %* ^> nul 2^>^&1 cmd /s /c "%* > nul 2>&1" goto :eof