[MERGE #2256 @dilijev] run_build.ps1: Refactor and fix some bugs with VSO pogo builds of ChakraCore.

Merge pull request #2256 from dilijev:iffnuget

* Fix inverted logic for running NuGet.exe.
* Fix incorrect $binpath fallback location.
* Refactor build scripts: rename confusingly-named variables, information hiding.

Build/scripts/compose_build.ps1

@@ -39,15 +39,15 @@ $changeInfo = (Get-Content $changeJson) -join "`n" | ConvertFrom-Json
 # between the partially-composed root and the metadata directories.
 # Exclude change.json and build.json, the results of a previous composition already in the root.
 Get-ChildItem -Path $rootPath "*.json" -Recurse `
-    | ? { -not ($_.Name -in @("change.json", "build.json")) } `
-    | % { Move-Item -Verbose -Force -Path $_.FullName -Destination $rootPath }
+    | Where-Object { -not ($_.Name -in @("change.json", "build.json")) } `
+    | ForEach-Object { Move-Item -Verbose -Force -Path $_.FullName -Destination $rootPath }
 
 # Determine the overall build status. Mark the build as "passed" until "failed" is encountered.
 $overallBuildStatus = "passed"
 
 $files = Get-ChildItem -Path $rootPath "*.json" -Recurse `
-    | ? { -not ($_.Name -in @("change.json", "build.json")) } `
-    | % { $_.FullName }
+    | Where-Object { -not ($_.Name -in @("change.json", "build.json")) } `
+    | ForEach-Object { $_.FullName }
 $builds = New-Object System.Collections.ArrayList
 foreach ($file in $files) {
     $json = (Get-Content $file) -join "`n" | ConvertFrom-Json

Build/scripts/finalize_build.ps1

@@ -23,11 +23,13 @@ param (
     $corePathSegment = "" # e.g. "core"
 )
 
-$sourcesDir = $Env:BUILD_SOURCESDIRECTORY
-$coreSourcesDir = Join-Path $sourcesDir $corePathSegment
+. $PSScriptRoot\pre_post_util.ps1
+
+$sourcesDir, $_, $_, $_ = `
+    ComputePaths `
+        -arch $arch -flavor $flavor -subtype $subtype -OuterScriptRoot $PSScriptRoot
 
-$OuterScriptRoot = $PSScriptRoot
-. "$PSScriptRoot\pre_post_util.ps1"
+$coreSourcesDir = Join-Path $sourcesDir $corePathSegment
 
 $buildName = ConstructBuildName -arch $arch -flavor $flavor -subtype $subtype
 
@@ -118,7 +120,7 @@ Copy-Item -Verbose -Force $buildFlavorJsonFile $metadataDir
 
 # Search for *.nuspec files and copy them to $metadataDir
 Get-ChildItem -Path (Join-Path $sourcesDir "Build") "*.nuspec" `
-    | % { Copy-Item -Verbose -Force $_.FullName $metadataDir }
+    | ForEach-Object { Copy-Item -Verbose -Force $_.FullName $metadataDir }
 
 #
 # Copy POGO directory if present for this build

Build/scripts/init_build.ps1

@@ -36,13 +36,14 @@ param (
     [string]$oauth
 )
 
+. $PSScriptRoot\pre_post_util.ps1
+
 #
-# Define values for variables based on parameters and environment variables
-# with default values in case the environment variables are not defined.
+# Process build type parameters
 #
 
-. $PSScriptRoot\util.ps1
-$gitExe = GetGitPath
+# Define values for variables based on parameters and environment variables
+# with default values in case the environment variables are not defined.
 
 $BuildType = UseValueOrDefault $buildtype $Env:BuildType
 $BuildPlatform = UseValueOrDefault $arch $Env:BuildPlatform
@@ -85,9 +86,11 @@ if ($BuildType) {
     exit 1
 }
 
-$CommitHash = UseValueOrDefault $Env:BUILD_SOURCEVERSION $(iex "${gitExe} rev-parse HEAD")
+$gitExe = GetGitPath
+
+$CommitHash = UseValueOrDefault $Env:BUILD_SOURCEVERSION $(Invoke-Expression "${gitExe} rev-parse HEAD")
 
-$branchFullName  = UseValueOrDefault $Env:BUILD_SOURCEBRANCH $(iex "${gitExe} rev-parse --symbolic-full-name HEAD")
+$branchFullName  = UseValueOrDefault $Env:BUILD_SOURCEBRANCH $(Invoke-Expression "${gitExe} rev-parse --symbolic-full-name HEAD")
 
 $SourcesDirectory = UseValueOrDefault $Env:BUILD_SOURCESDIRECTORY $(GetRepoRoot)
 $BinariesDirectory = UseValueOrDefault (Join-Path $SourcesDirectory "Build\VcBuild")
@@ -95,24 +98,18 @@ $ObjectDirectory = Join-Path $BinariesDirectory "obj\${BuildPlatform}_${BuildCon
 
 $DropRoot = UseValueOrDefault $dropRoot $Env:DROP_ROOT (Join-Path $(GetRepoRoot) "_DROP")
 
-# set up required variables and import pre_post_util.ps1
-$arch = $BuildPlatform
-$flavor = $BuildConfiguration
-$OuterScriptRoot = $PSScriptRoot # Used in pre_post_util.ps1
-. "$PSScriptRoot\pre_post_util.ps1"
-
 $BuildName = ConstructBuildName -arch $BuildPlatform -flavor $BuildConfiguration -subtype $BuildSubtype
 
 $BranchName = $branchFullName.split('/',3)[2]
 $BranchPath = $BranchName.replace('/','\')
 
 if (-not $CommitHash) {
-    $CommitHash = iex "${gitExe} rev-parse HEAD"
+    $CommitHash = Invoke-Expression "${gitExe} rev-parse HEAD"
 }
-$CommitShortHash = $(iex "${gitExe} rev-parse --short $CommitHash")
+$CommitShortHash = $(Invoke-Expression "${gitExe} rev-parse --short $CommitHash")
 
-$Username = (iex "${gitExe} log $CommitHash -1 --pretty=%ae").split('@')[0]
-$CommitDateTime = [DateTime]$(iex "${gitExe} log $CommitHash -1 --pretty=%aD")
+$Username = (Invoke-Expression "${gitExe} log $CommitHash -1 --pretty=%ae").split('@')[0]
+$CommitDateTime = [DateTime]$(Invoke-Expression "${gitExe} log $CommitHash -1 --pretty=%aD")
 $CommitTime = Get-Date $CommitDateTime -Format yyMMdd.HHmm
 
 #

Build/scripts/post_build.ps1

@@ -23,7 +23,7 @@ param (
     [string]$subtype = "default",
 
     [string]$srcpath = "",
-    [string]$binpath = "",
+    [string]$buildRoot = "",
     [string]$objpath = "",
 
     [Parameter(Mandatory=$True)]
@@ -44,29 +44,31 @@ param (
     [switch]$skipTests # or set $Env:SKIP_TESTS before invoking build
 )
 
+. $PSScriptRoot\pre_post_util.ps1
+
+$srcpath, $buildRoot, $objpath, $_ = `
+    ComputePaths `
+        -arch $arch -flavor $flavor -subtype $subtype -OuterScriptRoot $PSScriptRoot `
+        -srcpath $srcpath -buildRoot $buildRoot -objpath $objpath -binpath $_
+
 $skipTests = $skipTests -or (Test-Path Env:\SKIP_TESTS)
 
 $global:exitcode = 0
 
 if ($arch -eq "*") {
 
-    . "$PSScriptRoot\util.ps1"
     foreach ($arch in ("x86", "x64", "arm")) {
-        ExecuteCommand "$PSScriptRoot\post_build.ps1 -arch $arch -flavor $flavor -srcpath ""$srcpath"" -binpath ""$binpath"" -objpath ""$objpath"" -srcsrvcmdpath ""$srcsrvcmdpath"" -bvtcmdpath ""$bvtcmdpath"" -repo ""$repo""" -logFile ""$logFile""
+        ExecuteCommand "$PSScriptRoot\post_build.ps1 -arch $arch -flavor $flavor -srcpath ""$srcpath"" -buildRoot ""$buildRoot"" -objpath ""$objpath"" -srcsrvcmdpath ""$srcsrvcmdpath"" -bvtcmdpath ""$bvtcmdpath"" -repo ""$repo""" -logFile ""$logFile""
     }
 
 } elseif ($flavor -eq "*") {
 
-    . "$PSScriptRoot\util.ps1"
     foreach ($flavor in ("debug", "test", "release")) {
-        ExecuteCommand "$PSScriptRoot\post_build.ps1 -arch $arch -flavor $flavor -srcpath ""$srcpath"" -binpath ""$binpath"" -objpath ""$objpath"" -srcsrvcmdpath ""$srcsrvcmdpath"" -bvtcmdpath ""$bvtcmdpath"" -repo ""$repo""" -logFile ""$logFile""
+        ExecuteCommand "$PSScriptRoot\post_build.ps1 -arch $arch -flavor $flavor -srcpath ""$srcpath"" -buildRoot ""$buildRoot"" -objpath ""$objpath"" -srcsrvcmdpath ""$srcsrvcmdpath"" -bvtcmdpath ""$bvtcmdpath"" -repo ""$repo""" -logFile ""$logFile""
     }
 
 } else {
 
-    $OuterScriptRoot = $PSScriptRoot
-    . "$PSScriptRoot\pre_post_util.ps1"
-
     $buildName = ConstructBuildName -arch $arch -flavor $flavor -subtype $subtype
 
     if (($logFile -eq "") -and (Test-Path Env:\TF_BUILD_BINARIESDIRECTORY)) {
@@ -86,12 +88,12 @@ if ($arch -eq "*") {
     WriteMessage "BVT Command  : $bvtcmdpath"
     WriteMessage ""
 
-    $srcsrvcmd = ("{0} {1} {2} {3}\bin\{4}\*.pdb" -f $srcsrvcmdpath, $repo, $srcpath, $binpath, $buildName)
-    $prefastlog = ("{0}\logs\PrefastCheck.{1}.log" -f $binpath, $buildName)
+    $srcsrvcmd = ("{0} {1} {2} {3}\bin\{4}\*.pdb" -f $srcsrvcmdpath, $repo, $srcpath, $buildRoot, $buildName)
+    $prefastlog = ("{0}\logs\PrefastCheck.{1}.log" -f $buildRoot, $buildName)
     $prefastcmd = "$PSScriptRoot\check_prefast_error.ps1 -directory $objpath -logFile $prefastlog"
 
     # generate srcsrv
-    if ((Test-Path $srcsrvcmdpath) -and (Test-Path $srcpath) -and (Test-Path $binpath)) {
+    if ((Test-Path $srcsrvcmdpath) -and (Test-Path $srcpath) -and (Test-Path $buildRoot)) {
         ExecuteCommand($srcsrvcmd)
     }
 

Build/scripts/pre_build.ps1

@@ -45,21 +45,26 @@ param (
     [string]$oauth
 )
 
-$OuterScriptRoot = $PSScriptRoot # Used in pre_post_util.ps1
-. "$PSScriptRoot\pre_post_util.ps1"
+. $PSScriptRoot\pre_post_util.ps1
 
-if (($logFile -eq "") -and (Test-Path Env:\TF_BUILD_BINARIESDIRECTORY)) {
-    if (-not(Test-Path -Path "${Env:TF_BUILD_BINARIESDIRECTORY}\logs")) {
-        $dummy = New-Item -Path "${Env:TF_BUILD_BINARIESDIRECTORY}\logs" -ItemType Directory -Force
+$srcpath, $buildRoot, $objpath, $binpath = `
+    ComputePaths `
+        -arch $arch -flavor $flavor -subtype $subtype -OuterScriptRoot $PSScriptRoot `
+        -srcpath $srcpath -buildRoot $buildRoot -objpath $objpath -binpath $binpath
+
+WriteCommonArguments
+
+$buildName = ConstructBuildName $arch $flavor $subtype
+if (($logFile -eq "") -and (Test-Path $buildRoot)) {
+    if (-not(Test-Path -Path "${buildRoot}\logs")) {
+        $dummy = New-Item -Path "${buildRoot}\logs" -ItemType Directory -Force
     }
-    $logFile = "${Env:TF_BUILD_BINARIESDIRECTORY}\logs\pre_build.${Env:BuildName}.log"
+    $logFile = "${buildRoot}\logs\pre_build.${buildName}.log"
     if (Test-Path -Path $logFile) {
         Remove-Item $logFile -Force
     }
 }
 
-WriteCommonArguments
-
 #
 # Create packages.config files
 #
@@ -72,7 +77,7 @@ $packagesConfigFileText = @"
 "@
 
 $packagesFiles = Get-ChildItem -Path $Env:TF_BUILD_SOURCESDIRECTORY *.vcxproj -Recurse `
-    | % { Join-Path $_.DirectoryName "packages.config" }
+    | ForEach-Object { Join-Path $_.DirectoryName "packages.config" }
 
 foreach ($file in $packagesFiles) {
     if (-not (Test-Path $file)) {
@@ -91,7 +96,7 @@ if (Test-Path Env:\TF_BUILD_SOURCEGETVERSION)
 
     $CoreHash = ""
     if (Test-Path $corePath) {
-        $CoreHash = iex "$gitExe rev-parse ${commitHash}:core"
+        $CoreHash = Invoke-Expression "$gitExe rev-parse ${commitHash}:core"
         if (-not $?) {
             $CoreHash = ""
         }
@@ -112,7 +117,7 @@ if (Test-Path Env:\TF_BUILD_SOURCEGETVERSION)
 
     # commit message
     $command = "$gitExe log -1 --name-only -p $commitHash"
-    $CommitMessageLines = iex $command
+    $CommitMessageLines = Invoke-Expression $command
     $CommitMessage = $CommitMessageLines -join "`r`n"
 
     $changeTextFile = Join-Path -Path $outputDir -ChildPath "change.txt"
@@ -183,7 +188,7 @@ $CommitMessage
 </Project>
 "@
 
-    $propsFileContent = $propsFileTemplate -f $binpath, $objpath, $buildPushIdPart1, $buildPushIdPart2, $buildCommit, $buildDate
+    $propsFileContent = $propsFileTemplate -f $buildRoot, $objpath, $buildPushIdPart1, $buildPushIdPart2, $buildCommit, $buildDate
 
     Write-Output "-----"
     Write-Output $propsFile

Build/scripts/pre_post_util.ps1

@@ -3,10 +3,11 @@
 # Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 #-------------------------------------------------------------------------------------------------------
 
-. "$PSScriptRoot\util.ps1"
+. $PSScriptRoot\util.ps1
 
 function WriteCommonArguments() {
     WriteMessage "  Source Path: $srcpath"
+    WriteMessage "   Build Root: $buildRoot"
     WriteMessage "  Object Path: $objpath"
     WriteMessage "Binaries Path: $binpath"
 }
@@ -14,7 +15,7 @@ function WriteCommonArguments() {
 function GetVersionField($fieldname) {
     $gitExe = GetGitPath
     $query = "#define ${fieldname} (\d+)"
-    $line = (iex "${gitExe} grep -P ""${query}"" :/")
+    $line = (Invoke-Expression "${gitExe} grep -P ""${query}"" :/")
     $matches = $line | Select-String $query
     if ($matches) {
         return $matches[0].Matches.Groups[1].Value
@@ -25,7 +26,8 @@ function GetVersionField($fieldname) {
 function GetBuildInfo($oauth, $commitHash) {
     # Get the git remote path and construct the REST API URI
     $gitExe = GetGitPath
-    $remote = (iex "$gitExe remote -v" | ? { $_.contains("_git") })[0].split()[1].replace("_git", "_apis/git/repositories")
+    $remote = (Invoke-Expression "$gitExe remote -v" `
+        | Where-Object { $_.contains("_git") })[0].split()[1].replace("_git", "_apis/git/repositories")
     $remote = $remote.replace("mshttps", "https")
 
     # Get the pushId and push date time to use that for build number and build date time
@@ -46,7 +48,25 @@ function GetBuildPushId($info) {
     return @($buildPushId, $buildPushIdPart1, $buildPushIdPart2, $buildPushIdString)
 }
 
+function EnsureVariables($functionName, $arch, $flavor, $OuterScriptRoot) {
+    if (("$arch" -eq "") -or ("$flavor" -eq "") -or ("$OuterScriptRoot" -eq ""))
+    {
+        WriteErrorMessage @"
+
+        ${functionName}: Required variables not set:
+            `$arch = $arch
+            `$flavor = $flavor
+            `$OuterScriptRoot = $OuterScriptRoot
+
+"@
+
+        throw "Cannot continue: ${functionName}: required variables not set."
+    }
+}
+
 function ConstructBuildName($arch, $flavor, $subtype) {
+    EnsureVariables "ConstructBuildName" $arch $flavor "(OuterScriptRoot not needed)"
+
     if ($subtype -eq "codecoverage") {
         # TODO eliminate tools' dependency on this particular formatting exception
         # Normalize the $BuildName of even if the $BuildType is e.g. x64_test_codecoverage
@@ -58,22 +78,12 @@ function ConstructBuildName($arch, $flavor, $subtype) {
     }
 }
 
-# Compute paths
-
-if (("$arch" -eq "") -or ("$flavor" -eq "") -or ("$OuterScriptRoot" -eq ""))
-{
-    WriteErrorMessage @"
-
-    Required variables not set before script was included:
-        `$arch = $arch
-        `$flavor = $flavor
-        `$OuterScriptRoot = $OuterScriptRoot
-
-"@
-
-    throw "Cannot continue - required variables not set."
+function ComputePaths($arch, $flavor, $subtype, $OuterScriptRoot, $srcpath = "", $buildRoot = "", $objpath = "", $binpath = "") {
+    EnsureVariables "ComputePaths" $arch $flavor $OuterScriptRoot
+    $buildName = ConstructBuildName $arch $flavor $subtype
+    $srcpath = UseValueOrDefault $srcpath "$Env:TF_BUILD_SOURCESDIRECTORY" (Resolve-Path "${OuterScriptRoot}\..\..")
+    $buildRoot = UseValueOrDefault $buildRoot "$Env:BinariesDirectory" "$Env:TF_BUILD_BINARIESDIRECTORY" (Join-Path $srcpath "Build\VcBuild")
+    $objpath = UseValueOrDefault $objpath "$Env:TF_BUILD_BUILDDIRECTORY" (Join-Path $buildRoot "obj\${buildName}")
+    $binpath = Join-Path $buildRoot "bin\${buildName}"
+    return @($srcpath, $buildRoot, $objpath, $binpath)
 }
-
-$srcpath = UseValueOrDefault $srcpath "$env:TF_BUILD_SOURCESDIRECTORY" (Resolve-Path "$OuterScriptRoot\..\..")
-$objpath = UseValueOrDefault $objpath "$env:TF_BUILD_BUILDDIRECTORY" "${srcpath}\Build\VcBuild\obj\${arch}_${flavor}"
-$binpath = UseValueOrDefault $binpath "$env:TF_BUILD_BINARIESDIRECTORY" "${srcpath}\Build\VcBuild"

Build/scripts/run_build.ps1

@@ -21,7 +21,7 @@ param (
 
     [switch]$clean,
 
-    [string]$binDir = "", # will be inferred if not provided.
+    [string]$buildRoot = "", # will be inferred if not provided
     [string]$buildlogsSubdir = "buildlogs",
 
     # assume NuGet is on the path, otherwise the caller must specify an explicit path
@@ -44,20 +44,21 @@ param (
     [string]$binaryName = "ch.exe"
 )
 
-#
-# Configure logging
-#
-
-$OuterScriptRoot = $PSScriptRoot
 . $PSScriptRoot\pre_post_util.ps1
 . $PSScriptRoot\locate_msbuild.ps1
 
+# TODO (doilij) update all scripts that use pre_post_util.ps1 and rely on ComputeProjectPaths
+$_, $buildRoot, $_, $binpath = `
+    ComputePaths `
+        -arch $arch -flavor $flavor -subtype $subtype -OuterScriptRoot $PSScriptRoot `
+        -buildRoot $buildRoot -binpath $binpath
+
 $buildName = ConstructBuildName -arch $arch -flavor $flavor -subtype $subtype
 
-if (($logFile -eq "") -and (Test-Path Env:\TF_BUILD_BINARIESDIRECTORY)) {
-    $logFile = "${Env:TF_BUILD_BINARIESDIRECTORY}\logs\run_build.${Env:BuildName}.log"
-}
+$buildlogsPath = Join-Path $buildRoot $buildlogsSubdir
+$logFile = UseValueOrDefault $logFile (Join-Path $buildRoot "logs\run_build.${buildName}.log")
 
+# Clear the log file
 if (($logFile -ne "") -and (Test-Path $logFile)) {
     Remove-Item $logFile -Force
 }
@@ -66,7 +67,10 @@ if (($logFile -ne "") -and (Test-Path $logFile)) {
 # NuGet restore
 #
 
-if (-not (Get-Command $nugetExe -ErrorAction SilentlyContinue)) {
+# To support both local builds where NuGet's location is known,
+# and VSO builds where the location is not known and which use a NuGet restore task instead:
+# Run `nuget restore` IFF $nugetExe exists.
+if (Get-Command $nugetExe -ErrorAction SilentlyContinue) {
     ExecuteCommand "& $nugetExe restore $solutionFile -NonInteractive"
 }
 
@@ -80,21 +84,18 @@ if (-not $msbuildExe) {
     exit 1
 }
 
-$binDir = UseValueOrDefault "$binDir" "${Env:BinariesDirectory}" "${Env:BUILD_SOURCESDIRECTORY}\Build\VcBuild"
-$buildlogsPath = Join-Path $binDir $buildlogsSubdir
-
 $skipPogo = $skipPogo -or (Test-Path Env:\SKIP_POGO)
 
-# if $binpath is not set or if it is an invalid path, then infer it
-if ((-not $binpath) -or (-not (Test-Path $binpath))) {
-    $binpath = Join-Path $binDir "bin\${buildName}"
+# If $binpath is not set, then infer it. A missing path is okay because it will be created.
+if (-not $binpath) {
+    $binpath = Join-Path $buildRoot "bin\${buildName}"
 }
 
 $defaultParams = "$solutionFile /nologo /m /nr:false /p:platform=`"${arch}`" /p:configuration=`"${flavor}`""
 $loggingParams = @(
-    "/fl1 `"/flp1:logfile=${buildlogsPath}\build.${Env:BuildName}.log;verbosity=normal`"",
-    "/fl2 `"/flp2:logfile=${buildlogsPath}\build.${Env:BuildName}.err;errorsonly`"",
-    "/fl3 `"/flp3:logfile=${buildlogsPath}\build.${Env:BuildName}.wrn;warningsonly`"",
+    "/fl1 `"/flp1:logfile=${buildlogsPath}\build.${buildName}.log;verbosity=normal`"",
+    "/fl2 `"/flp2:logfile=${buildlogsPath}\build.${buildName}.err;errorsonly`"",
+    "/fl3 `"/flp3:logfile=${buildlogsPath}\build.${buildName}.wrn;warningsonly`"",
     "/verbosity:normal"
     ) -join " "
 
@@ -154,8 +155,8 @@ if ($subtype -eq "pogo") {
 if (("$binpath" -ne "") -and (Test-Path $binpath)) {
     # remove *.pgc, *.pgd, and pgort*
     Get-ChildItem -Recurse -Path $binpath "*" `
-        | ? { $_.Name -match "(.*\.pg[cd]|pgort.*)" } `
-        | % { Remove-Item -Force $_.FullName }
+        | Where-Object { $_.Name -match "(.*\.pg[cd]|pgort.*)" } `
+        | ForEach-Object { Remove-Item -Force $_.FullName }
 }
 
 exit $global:lastexitcode

Build/scripts/util.ps1

@@ -27,7 +27,7 @@ function GetGitPath() {
 
 function GetRepoRoot() {
     $gitExe = GetGitPath
-    return iex "$gitExe rev-parse --show-toplevel"
+    return Invoke-Expression "$gitExe rev-parse --show-toplevel"
 }
 
 function WriteMessage($str) {