diff --git a/.jenkins/cross-compilation/Jenkinsfile b/.jenkins/cross-compilation/Jenkinsfile index 2c70bb16b6..f54af0c4d5 100644 --- a/.jenkins/cross-compilation/Jenkinsfile +++ b/.jenkins/cross-compilation/Jenkinsfile @@ -13,19 +13,26 @@ pipeline { // Only allow one build at a time of this job. disableConcurrentBuilds(abortPrevious: true) + + // We will do checkout manually. + skipDefaultCheckout() } stages { - stage('Set build as pending') + stage('Set up workspace') { steps { - // Set the build status... + cleanWs(deleteDirs: true, + disableDeferredWipeout: true, + notFailBuild: true) + checkout scm + script { u = load '.jenkins/utils.groovy' - u.startBuild("Cross-compilation Tests"); + u.startCheck('Cross-compilation checks', 'Setting up workspace...') } // Create a directory for our resulting reports. @@ -79,6 +86,11 @@ pipeline } steps { + script + { + u.updateCheckStatus('Building mlpack for ' + env.target + '...') + } + sh ''' rm -rf build/ mkdir build/ @@ -95,6 +107,11 @@ pipeline make mlpack_test; ''' + script + { + u.updateCheckStatus('Testing mlpack on ' + env.target + '...') + } + withCredentials([sshUserPrivateKey( credentialsId: 'mlpack-jenkins-cross-compile-rsa-key', keyFileVariable: 'KEY_FILE', @@ -138,24 +155,27 @@ pipeline post { + success + { + script { u.finishCheck('Cross-compilation checks passed.', true) } + } + + failure + { + script { u.finishCheck('Cross-compilation checks failed.', false) } + } + always { - junit '**/reports/mlpack_test.*.junit.xml' + junit(allowEmptyResults: true, + skipPublishingChecks: true, + testResults: '**/reports/mlpack_test.*.junit.xml') // Clean the workspace. cleanWs(cleanWhenNotBuilt: true, deleteDirs: true, disableDeferredWipeout: true, notFailBuild: true) - - script - { - u.setBuildStatus(result: currentBuild.currentResult, - context: "Cross-compilation Tests", - successMessage: "Cross-compilation succeeded with no errors.", - unstableMessage: "Cross-compilation build unstable.", - failureMessage: "Cross-compilation failed."); - } } } } diff --git a/.jenkins/doc-link-check/Jenkinsfile b/.jenkins/doc-link-check/Jenkinsfile index d56e8be817..2a96f3bdcd 100644 --- a/.jenkins/doc-link-check/Jenkinsfile +++ b/.jenkins/doc-link-check/Jenkinsfile @@ -23,21 +23,37 @@ pipeline { // Only allow one build at a time of this job. disableConcurrentBuilds(abortPrevious: true) + + // We will do checkout manually. + skipDefaultCheckout() } stages { - // First we have to check out the jenkins-conf repository, which contains - // the scripts that we will use for checking the style. + // Check out the repository and start the check. + stage('Set up workspace') + { + steps + { + cleanWs(deleteDirs: true, + disableDeferredWipeout: true, + notFailBuild: true) + checkout scm + + script + { + u = load '.jenkins/utils.groovy' + u.startCheck('Documentation link check', 'Setting up workspace...') + } + } + } + + // Actually run the check. stage('Build documentation and check links') { steps { - script - { - u = load '.jenkins/utils.groovy' - u.startBuild("Documentation Link Check"); - } + script { u.updateCheckStatus('Checking documentation links...') } sh ''' # Set $HOME because the Docker container may be running with a @@ -76,32 +92,25 @@ pipeline post { + success { script { u.finishCheck('No HTML issues found.', true) } } + failure { script { u.finishCheck('Problems found in HTML.', false) } } + always { // Publish the generated HTML. publishHTML([ - allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll: true, - reportDir: 'doc/html/', - reportFiles: 'index.html', - reportName: 'Build documentation']); + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'doc/html/', + reportFiles: 'index.html', + reportName: 'Build documentation']); // Clean the workspace. cleanWs(cleanWhenNotBuilt: true, deleteDirs: true, disableDeferredWipeout: true, notFailBuild: true); - - // Set the build status. - script - { - u.setBuildStatus(result: currentBuild.currentResult, - context: "Documentation Link Check", - successMessage: "All HTML links checked.", - unstableMessage: "Problem with HTML links.", - failureMessage: "HTML link check failure or problem."); - } } } } diff --git a/.jenkins/doc-snippet-build/Jenkinsfile b/.jenkins/doc-snippet-build/Jenkinsfile index 6a27bc66ac..8452c1a170 100644 --- a/.jenkins/doc-snippet-build/Jenkinsfile +++ b/.jenkins/doc-snippet-build/Jenkinsfile @@ -18,19 +18,36 @@ pipeline { // Only allow one build at a time of this job. disableConcurrentBuilds(abortPrevious: true) + + // We will do checkout manually. + skipDefaultCheckout() } stages { + stage('Set up workspace') + { + steps + { + // Clean the workspace. + cleanWs(deleteDirs: true, + disableDeferredWipeout: true, + notFailBuild: true); + checkout scm + + script + { + u = load '.jenkins/utils.groovy' + u.startCheck('Documentation snippet build', 'Setting up workspace...') + } + } + } + stage('Extract and build documentation snippets') { steps { - script - { - u = load '.jenkins/utils.groovy' - u.startBuild("Documentation Snippet Build"); - } + script { u.updateCheckStatus('Testing documentation snippets...') } sh''' export CCACHE_DIR=/opt/ccache/; @@ -52,6 +69,22 @@ pipeline post { + success + { + script + { + u.finishCheck('All documentation snippets build and run.', true) + } + } + + failure + { + script + { + u.finishCheck('Problems building documentation snippets.', false) + } + } + always { // Clean the workspace. @@ -59,16 +92,6 @@ pipeline deleteDirs: true, disableDeferredWipeout: true, notFailBuild: true); - - // Set the build status. - script - { - u.setBuildStatus(result: currentBuild.currentResult, - context: "Documentation Snippet Build", - successMessage: "All snippets built and run successfully.", - unstableMessage: "Snippets build unstable..", - failureMessage: "Snippet build or runtime failure."); - } } } } diff --git a/.jenkins/memory-checks/Jenkinsfile b/.jenkins/memory-checks/Jenkinsfile index e24467d5e4..b2059b2337 100644 --- a/.jenkins/memory-checks/Jenkinsfile +++ b/.jenkins/memory-checks/Jenkinsfile @@ -17,22 +17,32 @@ pipeline { // Only allow one build at a time of this job. disableConcurrentBuilds(abortPrevious: true) + + // We will do checkout manually. + skipDefaultCheckout() } stages { // First we have to check out the jenkins-conf repository, which contains // the scripts that we will use for checking the style. - stage('Check out jenkins-conf repository') + stage('Set up workspace') { steps { + cleanWs(deleteDirs: true, + disableDeferredWipeout: true, + notFailBuild: true) + checkout scm + script { u = load '.jenkins/utils.groovy' - u.startBuild("Memory Checks"); + u.startCheck('Memory checks', 'Setting up workspace...') } + // We also need the jenkins-conf repository for the memory checking + // scripts. sh ''' git clone https://github.com/mlpack/jenkins-conf ''' @@ -44,6 +54,8 @@ pipeline { steps { + script { u.updateCheckStatus('Building mlpack...') } + sh ''' export CCACHE_DIR=/opt/ccache/; ccache --zero-stats @@ -69,6 +81,8 @@ pipeline // what files have changed. script { + u.updateCheckStatus('Running memory checks...') + if (env.BRANCH_NAME.startsWith('PR-')) { // Strip 'PR-' from the front. @@ -112,9 +126,20 @@ pipeline post { + success + { + script { u.finishCheck('No memory errors found.', true) } + } + + failure + { + script { u.finishCheck('Memory errors found.', false) } + } + always { junit(allowEmptyResults: true, + skipPublishingChecks: true, testResults: '**/reports/tests/*.xml') // Clean the workspace. @@ -122,16 +147,6 @@ pipeline deleteDirs: true, disableDeferredWipeout: true, notFailBuild: true) - - // Set the build status. - script - { - u.setBuildStatus(result: currentBuild.currentResult, - context: "Memory Checks", - successMessage: "No memory errors.", - unstableMessage: "Build unstable.", - failureMessage: "Memory check failure with valgrind."); - } } } } diff --git a/.jenkins/style-checks/Jenkinsfile b/.jenkins/style-checks/Jenkinsfile index 53e2ffc784..8452cdafe7 100644 --- a/.jenkins/style-checks/Jenkinsfile +++ b/.jenkins/style-checks/Jenkinsfile @@ -16,20 +16,28 @@ pipeline { // Only allow one build at a time of this job. disableConcurrentBuilds(abortPrevious: true) + + // We will do checkout manually. + skipDefaultCheckout() } stages { - // First we have to check out the jenkins-conf repository, which contains - // the scripts that we will use for checking the style. - stage('Check out jenkins-conf repository') + // Clean the workspace and check out the code. + stage('Set up workspace') { steps { + cleanWs(deleteDirs: true, + disableDeferredWipeout: true, + notFailBuild: true) + checkout scm + script { + // Set the initial status. u = load '.jenkins/utils.groovy' - u.startBuild('Style Checks') + u.startCheck('Style checks', 'Setting up workspace...') } sh ''' @@ -43,6 +51,8 @@ pipeline { steps { + script { u.updateCheckStatus('Checking code style...') } + sh ''' mkdir -p reports ./jenkins-conf/linter/lint.sh \ @@ -60,34 +70,22 @@ pipeline post { // Mark unstable builds as failed. - unstable - { - script - { - error "Style check failure." - } - } + unstable { script { u.finishCheck('Style checks failed.', false) } } + failure { script { u.finishCheck('Style checks failed.', false) } } + success { script { u.finishCheck('Style checks passed.', true) } } always { // Process the test results. junit(allowEmptyResults: true, + skipPublishingChecks: true, testResults: '**/reports/cpplint.junit.xml') - // Clean the workspace. - cleanWs(cleanWhenNotBuilt: false, + // Clean the workspace after the build too. + cleanWs(cleanWhenNotBuilt: true, deleteDirs: true, disableDeferredWipeout: true, notFailBuild: true) - - script - { - u.setBuildStatus(result: currentBuild.currentResult, - context: "Style Checks", - successMessage: "No style issues.", - unstableMessage: "Style issues found.", - failureMessage: "Style issues found."); - } } } } diff --git a/.jenkins/utils.groovy b/.jenkins/utils.groovy index 50c6d19d5a..6bbdda9ad4 100644 --- a/.jenkins/utils.groovy +++ b/.jenkins/utils.groovy @@ -1,66 +1,54 @@ -// A simple utility to mark the build as pending on Github. -def startBuild(String context) +// These variables are set to hold state between status check updates. +def name = '' +def status = '' +def time = 0 + +def startCheck(String name, String status) { - step([ - $class: "GitHubCommitStatusSetter", - reposSource: [$class: "ManuallyEnteredRepositorySource", - url: "https://github.com/mlpack/mlpack"], - contextSource: [$class: "ManuallyEnteredCommitContextSource", - context: context ], - errorHandlers: [[$class: "ChangingBuildStatusErrorHandler", - result: "UNSTABLE"]], - statusResultSource: [$class: "ConditionalStatusResultSource", - results: [[$class: "AnyBuildResult", - message: "Building...", - state: "PENDING"]]] - ]); + // Set module-level variables that we will retain as we build. + this.name = name + this.status = status + this.time = currentBuild.duration + + publishChecks(name: name, + status: 'IN_PROGRESS', + title: status, + text: status, + detailsURL: currentBuild.absoluteUrl + 'console') } -// A simple utility to set the build status on Github for a commit. -def setBuildStatus(Map paramsMap) +def updateCheckStatus(String status) { - // Extract arguments from the map. - def result = paramsMap.result; - def context = paramsMap.context; - def successMessage = paramsMap.successMessage; - def unstableMessage = paramsMap.unstableMessage; - def failureMessage = paramsMap.failureMessage; + def stepTime = (currentBuild.duration - this.time) / 1000.0 + this.status += ' (' + stepTime.toString() + 's)\n' + this.status += status + this.time = currentBuild.duration - def message = "(unknown Jenkins build result)"; - def state = "FAILURE"; - if (result == "FAILURE") + publishChecks(name: this.name, + status: 'IN_PROGRESS', + title: status, + text: this.status, + detailsURL: currentBuild.absoluteUrl + 'console') +} + +def finishCheck(String status, boolean success) +{ + def stepTime = (currentBuild.duration - this.time) / 1000.0 + this.status += ' (' + stepTime.toString() + 's)\n' + this.status += status + + if (!success) { - message = failureMessage; - } - else if (result == "UNSTABLE") - { - message = unstableMessage; - state = "UNSTABLE"; - } - else if (result == "SUCCESS") - { - message = successMessage; - state = "SUCCESS"; - } - else if (result == "ABORTED") - { - message = "Job aborted."; - state = "ERROR"; + this.status += '\n\n' + + 'Click \'view more details\' below to see failure details...'; } - step([ - $class: "GitHubCommitStatusSetter", - reposSource: [$class: "ManuallyEnteredRepositorySource", - url: "https://github.com/mlpack/mlpack"], - contextSource: [$class: "ManuallyEnteredCommitContextSource", - context: context ], - errorHandlers: [[$class: "ChangingBuildStatusErrorHandler", - result: "UNSTABLE"]], - statusResultSource: [$class: "ConditionalStatusResultSource", - results: [[$class: "AnyBuildResult", - message: message, - state: state]]] - ]); + publishChecks(name: this.name, + status: 'COMPLETED', + conclusion: success ? 'SUCCESS' : 'FAILURE', + title: status, + text: this.status, + detailsURL: currentBuild.absoluteUrl + 'testReport') } return this