[MERGE #133] Refactored netci.groovy (closes #50); simplified trigger regexes (fixes #119).

Merge pull request #133 from dilijev:jenkins
Fixes #119.
Simplified trigger regexes; refactored dailyRegex to script-wide constant.

Closes #50.
Refactored style checks into a helper closure and call it once to produce each config.
Refactored build task generation to minimize duplication of code.

netci.groovy

@@ -15,296 +15,153 @@ def msbuildTypeMap = [
     'release':'fre'
 ]
 
-// ----------------
-// INNER LOOP TASKS
-// ----------------
-
-// Generate the builds for debug and release, commit and PRJob
-[true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
-    ['x86', 'x64', 'arm'].each { buildArch -> // build these architectures
-        ['debug', 'test', 'release'].each { buildType -> // build these configurations
-            def config = "${buildArch}_${buildType}"
-
-            // Determine the name for the new job.
-            // params: Project, BaseTaskName, IsPullRequest (appends _prtest)
-            def jobName = Utilities.getFullJobName(project, config, isPR)
-
-            def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
-            def analysisConfig = buildType in ['release']
-
-            def buildScript = "call jenkins.buildone.cmd ${buildArch} ${buildType}"
-            buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
-            def testScript = "call jenkins.testone.cmd ${buildArch} ${buildType}"
-            def analysisScript = ".\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err"
-
-            // Create a new job with the specified name.  The brace opens a new closure
-            // and calls made within that closure apply to the newly created job.
-            def newJob = job(jobName) {
-                // This opens the set of build steps that will be run.
-                // This looks strange, but it is actually a method call, with a
-                // closure as a param, since Groovy allows method calls without parens.
-                // (Compare with '.each' method used above.)
-                steps {
-                    batchFile(buildScript) // run the parameter as a batch script
-                    if (testableConfig) {
-                        // The test script will only run if the build is successful
-                        // because Jenkins will notice the failure and stop before
-                        // executing any more build tasks.
-                        batchFile(testScript)
-                    }
-                    if (analysisConfig) {
-                        // For release builds we want to run code analysis checks
-                        powerShell(analysisScript)
-                    }
+def dailyRegex = 'dailies'
+
+// ---------------
+// HELPER CLOSURES
+// ---------------
+
+def CreateBuildTasks = { machine, configTag, buildExtra, testExtra, excludeConfigIf, nonDefaultTaskSetup ->
+    [true, false].each { isPR ->
+        ['x86', 'x64', 'arm'].each { buildArch ->
+            ['debug', 'test', 'release'].each { buildType ->
+                if (excludeConfigIf && excludeConfigIf(isPR, buildArch, buildType)) {
+                    return // early exit: we don't want to create a job for this configuration
                 }
-            }
-
-            Utilities.setMachineAffinity(newJob, 'Windows_NT') // Server 2012 R2
-
-            def msbuildType = msbuildTypeMap.get(buildType)
-            def msbuildFlavor = "build_${buildArch}${msbuildType}"
-
-            def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
-            archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
-
-            Utilities.addArchival(newJob, archivalString,
-                '', // no exclusions from archival
-                false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
-                false) // archiveOnlyIfSuccessful=false ~= archiveAlways
-
-            // This call performs remaining common job setup on the newly created job.
-            // This is used most commonly for simple inner loop testing.
-            // It does the following:
-            //   1. Sets up source control for the project.
-            //   2. Adds a push trigger if the job is a PR job
-            //   3. Adds a github PR trigger if the job is a PR job.
-            //      The optional context (label that you see on github in the PR checks) is added.
-            //      If not provided the context defaults to the job name.
-            //   4. Adds standard options for build retention and timeouts
-            //   5. Adds standard parameters for PR and push jobs.
-            //      These allow PR jobs to be used for simple private testing, for instance.
-            // See the documentation for this function to see additional optional parameters.
-            Utilities.simpleInnerLoopJobSetup(newJob, project, isPR, "Windows ${config}")
-        }
-    }
-}
-
-// -----------------
-// DAILY BUILD TASKS
-// -----------------
-
-// build and test on Windows 7 with VS 2013 (Dev12/MsBuild12)
-[true, false].each { isPR ->
-    ['x86', 'x64'].each { buildArch -> // we don't support ARM on Windows 7
-        ['debug', 'test', 'release'].each { buildType ->
-            def config = "daily_dev12_${buildArch}_${buildType}"
-
-            def jobName = Utilities.getFullJobName(project, config, isPR)
-
-            def testableConfig = buildType in ['debug', 'test']
-            def analysisConfig = buildType in ['release']
-
-            def buildScript = "call jenkins.buildone.cmd ${buildArch} ${buildType} msbuild12"
-            buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
-            def testScript = "call jenkins.testone.cmd ${buildArch} ${buildType}"
-            def analysisScript = ".\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err"
 
-            def newJob = job(jobName) {
-                steps {
-                    batchFile(buildScript)
-                    if (testableConfig) {
-                        batchFile(testScript)
-                    }
-                    if (analysisConfig) {
-                        powerShell(analysisScript)
+                def config = "${buildArch}_${buildType}"
+                config = (configTag == null) ? config : "${configTag}_${config}"
+
+                // params: Project, BaseTaskName, IsPullRequest (appends '_prtest')
+                def jobName = Utilities.getFullJobName(project, config, isPR)
+
+                def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
+                def analysisConfig = buildType in ['release']
+
+                def buildScript = "call jenkins.buildone.cmd ${buildArch} ${buildType}"
+                buildScript += buildExtra ?: ''
+                buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
+                def testScript = "call jenkins.testone.cmd ${buildArch} ${buildType}"
+                testScript += testExtra ?: ''
+                def analysisScript = ".\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err"
+
+                def newJob = job(jobName) {
+                    // This opens the set of build steps that will be run.
+                    // This looks strange, but it is actually a method call, with a
+                    // closure as a param, since Groovy allows method calls without parens.
+                    // (Compare with '.each' method used above.)
+                    steps {
+                        batchFile(buildScript) // run the parameter as if it were a batch file
+                        if (testableConfig) {
+                            batchFile(testScript)
+                        }
+                        if (analysisConfig) {
+                            powerShell(analysisScript)
+                        }
                     }
                 }
-            }
-
-            Utilities.setMachineAffinity(newJob, "Windows 7")
-
-            def msbuildType = msbuildTypeMap.get(buildType)
-            def msbuildFlavor = "build_${buildArch}${msbuildType}"
-
-            def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
-            archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
 
-            Utilities.addArchival(newJob, archivalString,
-                '', // no exclusions from archival
-                false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
-                false) // archiveOnlyIfSuccessful=false ~= archiveAlways
-
-            Utilities.standardJobSetup(newJob, project, isPR)
-
-            if (isPR) {
-                // The addition of the trigger phrase will make the job "non-default" in Github.
-                def dailyRegex = '(dailies|dailys?)'
-                def groupRegex = '(win(dows)?\\s*7|(dev|msbuild)\\s*12)'
-                def triggerName = "Windows 7 ${config}"
-                def triggerRegex = "(${dailyRegex}|${groupRegex}|${triggerName})"
-                Utilities.addGithubPRTrigger(newJob,
-                    triggerName, // GitHub task name
-                    "(?i).*test\\W+${triggerRegex}.*")
-            } else {
-                Utilities.addPeriodicTrigger(newJob, '@daily')
+                Utilities.setMachineAffinity(newJob, machine)
+
+                def msbuildType = msbuildTypeMap.get(buildType)
+                def msbuildFlavor = "build_${buildArch}${msbuildType}"
+
+                def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
+                archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
+
+                Utilities.addArchival(newJob, archivalString,
+                    '', // no exclusions from archival
+                    false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
+                    false) // archiveOnlyIfSuccessful=false ~= archiveAlways
+
+                if (nonDefaultTaskSetup == null)
+                {
+                    // This call performs remaining common job setup on the newly created job.
+                    // This is used most commonly for simple inner loop testing.
+                    // It does the following:
+                    //   1. Sets up source control for the project.
+                    //   2. Adds a push trigger if the job is a PR job
+                    //   3. Adds a github PR trigger if the job is a PR job.
+                    //      The optional context (label that you see on github in the PR checks) is added.
+                    //      If not provided the context defaults to the job name.
+                    //   4. Adds standard options for build retention and timeouts
+                    //   5. Adds standard parameters for PR and push jobs.
+                    //      These allow PR jobs to be used for simple private testing, for instance.
+                    // See the documentation for this function to see additional optional parameters.
+                    Utilities.simpleInnerLoopJobSetup(newJob, project, isPR, "Windows ${config}")
+                }
+                else
+                {
+                    Utilities.standardJobSetup(newJob, project, isPR)
+                    nonDefaultTaskSetup(newJob, isPR, config)
+                }
             }
         }
     }
 }
 
-// build and test on the usual configuration (VS 2015) with -includeSlow
-[true, false].each { isPR ->
-    ['x86', 'x64', 'arm'].each { buildArch ->
-        ['debug', 'test', 'release'].each { buildType ->
-            def config = "daily_slow_${buildArch}_${buildType}"
-
-            def jobName = Utilities.getFullJobName(project, config, isPR)
-
-            def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
-            def analysisConfig = buildType in ['release']
-
-            def buildScript = "call jenkins.buildone.cmd ${buildArch} ${buildType}"
-            buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
-            def testScript = "call jenkins.testone.cmd ${buildArch} ${buildType}"
-            testScript += ' -includeSlow' // run slow tests
-            def analysisScript = ".\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err"
-
-            def newJob = job(jobName) {
-                steps {
-                    batchFile(buildScript)
-                    if (testableConfig) {
-                        batchFile(testScript)
-                    }
-                    if (analysisConfig) {
-                        powerShell(analysisScript)
-                    }
-                }
-            }
-
-            Utilities.setMachineAffinity(newJob, "Windows_NT") // Server 2012 R2
-
-            def msbuildType = msbuildTypeMap.get(buildType)
-            def msbuildFlavor = "build_${buildArch}${msbuildType}"
-
-            def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
-            archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
-
-            Utilities.addArchival(newJob, archivalString,
-                '', // no exclusions from archival
-                false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
-                false) // archiveOnlyIfSuccessful=false ~= archiveAlways
-
-            Utilities.standardJobSetup(newJob, project, isPR)
-
-            if (isPR) {
-                // The addition of the trigger phrase will make the job "non-default" in Github.
-                def dailyRegex = '(dailies|dailys?)'
-                def groupRegex = '(slow\\s*(tests?)?)'
-                def triggerName = "Windows ${config}"
-                def triggerRegex = "(${dailyRegex}|${groupRegex}|${triggerName})"
-                Utilities.addGithubPRTrigger(newJob,
-                    triggerName, // GitHub task name
-                    "(?i).*test\\W+${triggerRegex}.*")
-            } else {
-                Utilities.addPeriodicTrigger(newJob, '@daily')
-            }
-        }
+def DailyBuildTaskSetup = { newJob, isPR, triggerName, groupRegex ->
+    // The addition of triggers makes the job non-default in GitHub.
+    if (isPR) {
+        def triggerRegex = "(${dailyRegex}|${groupRegex}|${triggerName})"
+        Utilities.addGithubPRTrigger(newJob,
+            triggerName, // GitHub task name
+            "(?i).*test\\W+${triggerRegex}.*")
+    } else {
+        Utilities.addPeriodicTrigger(newJob, '@daily')
     }
 }
 
-// build and test on the usual configuration (VS 2015) with JIT disabled
-[true, false].each { isPR ->
-    ['x86', 'x64', 'arm'].each { buildArch ->
-        ['debug', 'test', 'release'].each { buildType ->
-            def config = "daily_disablejit_${buildArch}_${buildType}"
-
-            def jobName = Utilities.getFullJobName(project, config, isPR)
-
-            def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
-            def analysisConfig = buildType in ['release']
-
-            def buildScript = "call jenkins.buildone.cmd ${buildArch} ${buildType}"
-            buildScript += ' "/p:BuildJIT=false"' // don't build JIT on this build task
-            buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
-            def testScript = "call jenkins.testone.cmd ${buildArch} ${buildType}"
-            testScript += ' -disablejit' // don't run tests which would require JIT to be built
-            def analysisScript = ".\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err"
+def CreateStyleCheckTasks = { taskString, taskName, checkName ->
+    [true, false].each { isPR ->
+        def jobName = Utilities.getFullJobName(project, taskName, isPR)
 
-            def newJob = job(jobName) {
-                steps {
-                    batchFile(buildScript)
-                    if (testableConfig) {
-                        batchFile(testScript)
-                    }
-                    if (analysisConfig) {
-                        powerShell(analysisScript)
-                    }
-                }
-            }
-
-            Utilities.setMachineAffinity(newJob, "Windows_NT") // Server 2012 R2
-
-            def msbuildType = msbuildTypeMap.get(buildType)
-            def msbuildFlavor = "build_${buildArch}${msbuildType}"
-
-            def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
-            archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
-
-            Utilities.addArchival(newJob, archivalString,
-                '', // no exclusions from archival
-                false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
-                false) // archiveOnlyIfSuccessful=false ~= archiveAlways
-
-            Utilities.standardJobSetup(newJob, project, isPR)
-
-            if (isPR) {
-                // The addition of the trigger phrase will make the job "non-default" in Github.
-                def dailyRegex = '(dailies|dailys?)'
-                def groupRegex = '(disablejit|no(build)?jit|buildjit=false)'
-                def triggerName = "Windows ${config}"
-                def triggerRegex = "(${dailyRegex}|${groupRegex}|${triggerName})"
-                Utilities.addGithubPRTrigger(newJob,
-                    triggerName, // GitHub task name
-                    "(?i).*test\\W+${triggerRegex}.*")
-            } else {
-                Utilities.addPeriodicTrigger(newJob, '@daily')
+        def newJob = job(jobName) {
+            label('ubuntu')
+            steps {
+                shell(taskString)
             }
         }
+
+        Utilities.simpleInnerLoopJobSetup(newJob, project, isPR, checkName)
     }
 }
 
 // ----------------
-// CODE STYLE TASKS
+// INNER LOOP TASKS
 // ----------------
 
-// Create a job to check that no mixed line endings have been introduced.
-// Note: it is not necessary to run this job daily.
-[true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
-    def jobName = Utilities.getFullJobName(project, 'ubuntu_check_eol', isPR)
+CreateBuildTasks('Windows_NT', null, null, null, null, null)
 
-    def taskString = './jenkins.check_eol.sh'
-    def newJob = job(jobName) {
-        label('ubuntu')
-        steps {
-            shell(taskString)
-        }
-    }
+// -----------------
+// DAILY BUILD TASKS
+// -----------------
 
-    Utilities.simpleInnerLoopJobSetup(newJob, project, isPR, "EOL Check")
-}
+// build and test on Windows 7 with VS 2013 (Dev12/MsBuild12)
+CreateBuildTasks('Windows 7', 'daily_dev12', ' msbuild12', null,
+    /* excludeConfigIf */ { isPR, buildArch, buildType -> (buildArch == 'arm') },
+    /* nonDefaultTaskSetup */ { newJob, isPR, config ->
+        DailyBuildTaskSetup(newJob, isPR,
+            "Windows 7 ${config}",
+            '(win(dows)?\\s*7|(dev|msbuild)\\s*12)')})
 
-// Create a job to check that all source files have the correct Microsoft copyright notice.
-// Note: it is not necessary to run this job daily.
-[true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
-    def jobName = Utilities.getFullJobName(project, 'ubuntu_check_copyright', isPR)
+// build and test on the usual configuration (VS 2015) with -includeSlow
+CreateBuildTasks('Windows_NT', 'daily_slow', null, ' -includeSlow', null,
+    /* nonDefaultTaskSetup */ { newJob, isPR, config ->
+        DailyBuildTaskSetup(newJob, isPR,
+            "Windows ${config}",
+            '(slow\\s*(tests)?)')})
 
-    def taskString = './jenkins.check_copyright.sh'
-    def newJob = job(jobName) {
-        label('ubuntu')
-        steps {
-            shell(taskString)
-        }
-    }
+// build and test on the usual configuration (VS 2015) with JIT disabled
+CreateBuildTasks('Windows_NT', 'daily_disablejit', ' "/p:BuildJIT=false"', ' -disablejit', null,
+    /* nonDefaultTaskSetup */ { newJob, isPR, config ->
+        DailyBuildTaskSetup(newJob, isPR,
+            "Windows ${config}",
+            '(disablejit|nojit)')})
 
-    Utilities.simpleInnerLoopJobSetup(newJob, project, isPR, "Copyright Check")
-}
+// ----------------
+// CODE STYLE TASKS
+// ----------------
+
+CreateStyleCheckTasks('./jenkins.check_eol.sh', 'ubuntu_check_eol', 'EOL Check')
+CreateStyleCheckTasks('./jenkins.check_copyright.sh', 'ubuntu_check_copyright', 'Copyright Check')