[MERGE #4876 @dilijev] Provide more information when unit tests and native tests fail

Merge pull request #4876 from dilijev:native-exit-redux

Replaces #2043

test/jenkins.testall.cmd

@@ -61,8 +61,15 @@ set _HadFailures=0
   call :summarizeLogs
 
   echo.
-  if "%_HadFailures%" == "1" (
-    echo -- jenkins.testall.cmd ^>^> Tests failed! 1>&2
+  echo -- jenkins.testall.cmd ^>^> Failure code: %_HadFailures%
+  if "%_HadFailures%" NEQ "0" (
+    if "%_HadFailures%" == "3" (
+      echo -- jenkins.testall.cmd ^>^> Unit tests failed! 1>&2
+    ) else if "%_HadFailures%" == "4" (
+      echo -- jenkins.testall.cmd ^>^> Native tests failed! 1>&2
+    ) else (
+      echo -- jenkins.testall.cmd ^>^> Unknown failure! 1>&2
+    )
   ) else (
     echo -- jenkins.testall.cmd ^>^> Tests passed!
   )
@@ -80,7 +87,11 @@ set _HadFailures=0
 
   call :do %_TestDir%\runtests.cmd -%1 -quiet -cleanupall -nottags exclude_jenkins %_ExtraTestArgs% -binDir %_BinDir%
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  set _error=%ERRORLEVEL%
+  if "%_error%" NEQ "0" (
+    echo -- jenkins.testall.cmd ^>^> runtests.cmd returned error code: %_error%
+    set _HadFailures=3
+  )
 
   goto :eof
 
@@ -89,12 +100,20 @@ set _HadFailures=0
 :: ============================================================================
 :runNativeTests
 
-  call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% > %_LogDir%\nativetests.log 2>&1
+  set _LogFile=%_LogDir%\nativetests.log
+  call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% > %_LogFile% 2>&1
+
+  set _error=%ERRORLEVEL%
+  echo -- jenkins.testall.cmd ^>^> runnativetests.cmd returned error code: %_error%
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  if "%_error%" NEQ "0" (
+    echo -- jenkins.testall.cmd ^>^> runnativetests.cmd failed; printing %_LogFile%
+    powershell -c "if (Test-Path %_LogFile%) { Get-Content  %_LogFile% }"
+    set _HadFailures=4
+  )
 
   goto :eof
-  
+
 :: ============================================================================
 :: Summarize the logs into a listing of only the failures
 :: ============================================================================

test/jenkins.testone.cmd

@@ -30,6 +30,7 @@ setlocal
 
 set _RootDir=%~dp0..
 set _HadFailures=0
+set _error=0
 
 :: ============================================================================
 :: Main script
@@ -45,6 +46,7 @@ set _HadFailures=0
   set _TestDir=%CD%
 
   call jenkins.parsetestargs.cmd %*
+
   set _LogDir=%_TestDir%\logs\%_TestArch%_%_TestType%
   set _TestArgs=%_TestArch%%_TestType%
   set _BinDir=%_RootDir%\Build\VcBuild%_SpecialBuild%\bin
@@ -61,8 +63,15 @@ set _HadFailures=0
   call :summarizeLogs
 
   echo.
-  if "%_HadFailures%" == "1" (
-    echo -- jenkins.testone.cmd ^>^> Tests failed! 1>&2
+  echo -- jenkins.testone.cmd ^>^> Failure code: %_HadFailures%
+  if "%_HadFailures%" NEQ "0" (
+    if "%_HadFailures%" == "3" (
+      echo -- jenkins.testone.cmd ^>^> Unit tests failed! 1>&2
+    ) else if "%_HadFailures%" == "4" (
+      echo -- jenkins.testone.cmd ^>^> Native tests failed! 1>&2
+    ) else (
+      echo -- jenkins.testone.cmd ^>^> Unknown failure! 1>&2
+    )
   ) else (
     echo -- jenkins.testone.cmd ^>^> Tests passed!
   )
@@ -80,7 +89,10 @@ set _HadFailures=0
 
   call :do %_TestDir%\runtests.cmd -%1 -quiet -cleanupall -nottags exclude_jenkins %_ExtraTestArgs% -binDir %_BinDir%
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  if "%_error%" NEQ "0" (
+    echo -- jenkins.testone.cmd ^>^> runtests.cmd failed
+    set _HadFailures=3
+  )
 
   goto :eof
 
@@ -90,10 +102,17 @@ set _HadFailures=0
 :runNativeTests
 
   echo -- jenkins.testone.cmd ^>^> Running native tests... (this can take some time)
-  call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% > %_LogDir%\nativetests.log 2>&1
+  :: ensure the _LogDir exists
+  mkdir %_LogDir%
+  set _LogFile=%_LogDir%\nativetests.log
+  call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% > %_LogFile% 2>&1
   echo -- jenkins.testone.cmd ^>^> Running native tests... DONE!
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  if "%_error%" NEQ "0" (
+    echo -- jenkins.testone.cmd ^>^> runnativetests.cmd failed; printing %_LogFile%
+    powershell "if (Test-Path %_LogFile%) { Get-Content %_LogFile% }"
+    set _HadFailures=4
+  )
 
   goto :eof
 
@@ -106,6 +125,8 @@ set _HadFailures=0
   findstr /sp failed rl.results.log > summary.log
   findstr /sip failed nativetests.log >> summary.log
   rem Echo to stderr so that VSO includes the output in the build summary
+
+  echo -- jenkins.testone.cmd ^>^> Printing summary...
   type summary.log 1>&2
   popd
 
@@ -114,8 +135,9 @@ set _HadFailures=0
 :: ============================================================================
 :do
 
-  echo -- jenkins.testone.cmd ^>^> %*
+  echo -- jenkins.testone.cmd ^>^> :do %*
   cmd /s /c "%*"
+  set _error=%ERRORLEVEL%
 
   goto :eof
 
@@ -125,7 +147,8 @@ set _HadFailures=0
 :: ============================================================================
 :doSilent
 
-  echo -- jenkins.testone.cmd ^>^> %* ^> nul 2^>^&1
+  echo -- jenkins.testone.cmd ^>^> :doSilent %* ^> nul 2^>^&1
   cmd /s /c "%* > nul 2>&1"
+  set _error=%ERRORLEVEL%
 
   goto :eof

test/runcitests.cmd

@@ -86,7 +86,7 @@ set _HadFailures=0
     call :runNativeTests x86 test
     call :runNativeTests x64 debug
     call :runNativeTests x64 test
-    
+
     call :summarizeLogs summary.log
   ) else (
     call :runTests %_BuildArch% %_BuildType% %_ExtraArgs%
@@ -97,11 +97,19 @@ set _HadFailures=0
   call :copyLogsToDrop
 
   echo.
-  if "%_HadFailures%" == "1" (
-    echo -- runcitests.cmd ^>^> Tests failed! 1>&2
+  echo -- runcitests.cmd ^>^> Failure code: %_HadFailures%
+  if "%_HadFailures%" NEQ "0" (
+    if "%_HadFailures%" == "3" (
+      echo -- runcitests.cmd ^>^> Unit tests failed! 1>&2
+    ) else if "%_HadFailures%" == "4" (
+      echo -- runcitests.cmd ^>^> Native tests failed! 1>&2
+    ) else (
+      echo -- runcitests.cmd ^>^> Unknown failure! 1>&2
+    )
   ) else (
     echo -- runcitests.cmd ^>^> Tests passed!
   )
+
   echo -- runcitests.cmd ^>^> Logs at %_DropRootDir%\testlogs
 
   popd
@@ -123,7 +131,10 @@ set _HadFailures=0
 
   call :do %_TestDir%\runtests.cmd -%1%2 %3 -quiet -cleanupall -binDir %_StagingDir%\bin
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  if %ERRORLEVEL% NEQ 0 (
+    echo -- runcitests.cmd ^>^> runtests.cmd failed
+    set _HadFailures=3
+  )
 
   goto :eof
 
@@ -132,12 +143,17 @@ set _HadFailures=0
 :: ============================================================================
 :runNativeTests
 
-  call :do %_TestDir%\runnativetests.cmd -%1%2 > %_TestDir%\logs\%1_%2\nativetests.log 2>&1
+  set _LogFile=%_TestDir%\logs\%1_%2\nativetests.log
+  call :do %_TestDir%\runnativetests.cmd -%1%2 > %_LogFile% 2>&1
 
-  if ERRORLEVEL 1 set _HadFailures=1
+  if %ERRORLEVEL% NEQ 0 (
+    echo -- runcitests.cmd ^>^> runnativetests.cmd failed (printing %_LogFile% below)
+    powershell "if (Test-Path %_LogFile%) { Get-Content %_LogFile% }"
+    set _HadFailures=4
+  )
 
   goto :eof
-  
+
 :: ============================================================================
 :: Copy all result logs to the drop share
 :: ============================================================================

test/runnativetests.cmd

@@ -52,7 +52,6 @@ goto :main
 :: ============================================================================
 :main
 
-
   call :initVars
   call :parseArgs %*
 
@@ -73,9 +72,10 @@ goto :main
   call :copyScriptsAndBinaries
 
   call :runtest
-  
+
   call :cleanup
 
+  echo -- runnativetests.cmd ^>^> Exiting with exit code %_HadFailures%
   exit /b %_HadFailures%
 
 :: ============================================================================
@@ -171,7 +171,7 @@ goto :main
 :copyScriptsAndBinaries
   echo -- runnativetests.cmd ^>^> copying scripts from '%_RootDir%\bin\nativetests\Scripts' to '%_TestTempDir%'
   copy /y %_RootDir%\bin\nativetests\Scripts\*.js %_TestTempDir%
-  
+
   copy /y %_BinDir%ChakraCore.dll %_TestTempDir%
   copy /y %_BinDir%nativetests.exe %_TestTempDir%
 
@@ -182,9 +182,13 @@ goto :main
 :: ============================================================================
 :runtest
   pushd %_TestTempDir%
-  echo Calling %cd%\nativetests.exe with additional args: %_NativeTestArgs%
+  echo -- runnativetests.cmd ^>^> Calling %cd%\nativetests.exe with additional args: %_NativeTestArgs%
   call :do nativetests.exe %_NativeTestArgs%
-  if ERRORLEVEL 1 set _HadFailures=1
+  if "%_error%" NEQ "0" (
+    set _HadFailures=1
+  )
+  echo -- runnativetests.cmd ^>^> nativetests.exe exited with non-zero exit code (%_error%)
+
   popd
 
   goto :eof
@@ -206,6 +210,7 @@ goto :main
 
   echo ^>^> %*
   cmd /s /c "%*"
+  set _error=%ERRORLEVEL%
 
   goto :eof
 

test/runtests.cmd

@@ -119,6 +119,14 @@ goto :main
     )
   )
 
+  if "%_HadFailures%" NEQ "0" (
+    echo ^>^> Tests failed. See logs for details.
+  ) else (
+    echo ^>^> All tests passed!
+  )
+
+  echo ^>^> runtests.cmd -- exiting with exit code %_HadFailures%
+
   exit /b %_HadFailures%
 
 :: ============================================================================
@@ -224,7 +232,7 @@ goto :main
     set _Variants=lite
     goto :ArgOk
   )
-  
+
   if /i "%1" == "-nightly" (
     set _nightly=1
     if "%_ExtraVariants%" == "" (