apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Minor CI cleanups (#3911)

Browse Source 
* Add a test case that will fail to see what output we get in CI.

* Try to use the Github Checks API plugin to report status.

* Work a little harder with the publishChecks function.

* Hopefully fix URL path.

* Try one more place.

* Try to clean up the job a little bit; it does seem like it's owrking.

* Try to move checks definition into utils file.

* Maybe it needs to be in a script block.

* We can't publish a check until after the repo is cloned.

* Try to refactor the rest of the Jenkinsfile.

* Try to get the amount of time each step took.

* Oops, using the wrong variable for how long things have taken.

* Try and use the builtin checks only; maybe they will work.

* Revert "Try and use the builtin checks only; maybe they will work."

This reverts commit 461ea584206fcb12a4fc00cbcdb0d289a0bc7ef2.

* Try to clean up the output a little bit.

* Try to disable the extra check being published.

* Adapt the rest of the jobs to use the checks API.

* Fix syntactical issues.

* Remove failing test.

* Try adding a def to stop the warning.

* Okay, that's not the right syntax.

* Oops, remove old status setting block.
This commit is contained in:
Ryan Curtin
2025-03-30 09:25:10 -04:00
committed by GitHub
parent aa613f8b2f
commit 1ca98f966c
6 changed files with 193 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.jenkins/cross-compilation/Jenkinsfile
Vendored
+33 -13
View File
@@ -13,19 +13,26 @@ pipeline
{
// Only allow one build at a time of this job.
disableConcurrentBuilds(abortPrevious: true)
// We will do checkout manually.
skipDefaultCheckout()
}
stages
{
stage('Set build as pending')
stage('Set up workspace')
{
steps
{
// Set the build status...
cleanWs(deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
checkout scm
script
{
u = load '.jenkins/utils.groovy'
u.startBuild("Cross-compilation Tests");
u.startCheck('Cross-compilation checks', 'Setting up workspace...')
}
// Create a directory for our resulting reports.
@@ -79,6 +86,11 @@ pipeline
}
steps
{
script
{
u.updateCheckStatus('Building mlpack for ' + env.target + '...')
}
sh '''
rm -rf build/
mkdir build/
@@ -95,6 +107,11 @@ pipeline
make mlpack_test;
'''
script
{
u.updateCheckStatus('Testing mlpack on ' + env.target + '...')
}
withCredentials([sshUserPrivateKey(
credentialsId: 'mlpack-jenkins-cross-compile-rsa-key',
keyFileVariable: 'KEY_FILE',
@@ -138,24 +155,27 @@ pipeline
post
{
success
{
script { u.finishCheck('Cross-compilation checks passed.', true) }
}
failure
{
script { u.finishCheck('Cross-compilation checks failed.', false) }
}
always
{
junit '**/reports/mlpack_test.*.junit.xml'
junit(allowEmptyResults: true,
skipPublishingChecks: true,
testResults: '**/reports/mlpack_test.*.junit.xml')
// Clean the workspace.
cleanWs(cleanWhenNotBuilt: true,
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
script
{
u.setBuildStatus(result: currentBuild.currentResult,
context: "Cross-compilation Tests",
successMessage: "Cross-compilation succeeded with no errors.",
unstableMessage: "Cross-compilation build unstable.",
failureMessage: "Cross-compilation failed.");
}
}
}
}
.jenkins/doc-link-check/Jenkinsfile
Vendored
+32 -23
View File
@@ -23,21 +23,37 @@ pipeline
{
// Only allow one build at a time of this job.
disableConcurrentBuilds(abortPrevious: true)
// We will do checkout manually.
skipDefaultCheckout()
}
stages
{
// First we have to check out the jenkins-conf repository, which contains
// the scripts that we will use for checking the style.
// Check out the repository and start the check.
stage('Set up workspace')
{
steps
{
cleanWs(deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
checkout scm
script
{
u = load '.jenkins/utils.groovy'
u.startCheck('Documentation link check', 'Setting up workspace...')
}
}
}
// Actually run the check.
stage('Build documentation and check links')
{
steps
{
script
{
u = load '.jenkins/utils.groovy'
u.startBuild("Documentation Link Check");
}
script { u.updateCheckStatus('Checking documentation links...') }
sh '''
# Set $HOME because the Docker container may be running with a
@@ -76,32 +92,25 @@ pipeline
post
{
success { script { u.finishCheck('No HTML issues found.', true) } }
failure { script { u.finishCheck('Problems found in HTML.', false) } }
always
{
// Publish the generated HTML.
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: true,
reportDir: 'doc/html/',
reportFiles: 'index.html',
reportName: 'Build documentation']);
allowMissing: false,
alwaysLinkToLastBuild: false,
keepAll: true,
reportDir: 'doc/html/',
reportFiles: 'index.html',
reportName: 'Build documentation']);
// Clean the workspace.
cleanWs(cleanWhenNotBuilt: true,
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true);
// Set the build status.
script
{
u.setBuildStatus(result: currentBuild.currentResult,
context: "Documentation Link Check",
successMessage: "All HTML links checked.",
unstableMessage: "Problem with HTML links.",
failureMessage: "HTML link check failure or problem.");
}
}
}
}
.jenkins/doc-snippet-build/Jenkinsfile
Vendored
+38 -15
View File
@@ -18,19 +18,36 @@ pipeline
{
// Only allow one build at a time of this job.
disableConcurrentBuilds(abortPrevious: true)
// We will do checkout manually.
skipDefaultCheckout()
}
stages
{
stage('Set up workspace')
{
steps
{
// Clean the workspace.
cleanWs(deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true);
checkout scm
script
{
u = load '.jenkins/utils.groovy'
u.startCheck('Documentation snippet build', 'Setting up workspace...')
}
}
}
stage('Extract and build documentation snippets')
{
steps
{
script
{
u = load '.jenkins/utils.groovy'
u.startBuild("Documentation Snippet Build");
}
script { u.updateCheckStatus('Testing documentation snippets...') }
sh'''
export CCACHE_DIR=/opt/ccache/;
@@ -52,6 +69,22 @@ pipeline
post
{
success
{
script
{
u.finishCheck('All documentation snippets build and run.', true)
}
}
failure
{
script
{
u.finishCheck('Problems building documentation snippets.', false)
}
}
always
{
// Clean the workspace.
@@ -59,16 +92,6 @@ pipeline
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true);
// Set the build status.
script
{
u.setBuildStatus(result: currentBuild.currentResult,
context: "Documentation Snippet Build",
successMessage: "All snippets built and run successfully.",
unstableMessage: "Snippets build unstable..",
failureMessage: "Snippet build or runtime failure.");
}
}
}
}
.jenkins/memory-checks/Jenkinsfile
Vendored
+27 -12
View File
@@ -17,22 +17,32 @@ pipeline
{
// Only allow one build at a time of this job.
disableConcurrentBuilds(abortPrevious: true)
// We will do checkout manually.
skipDefaultCheckout()
}
stages
{
// First we have to check out the jenkins-conf repository, which contains
// the scripts that we will use for checking the style.
stage('Check out jenkins-conf repository')
stage('Set up workspace')
{
steps
{
cleanWs(deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
checkout scm
script
{
u = load '.jenkins/utils.groovy'
u.startBuild("Memory Checks");
u.startCheck('Memory checks', 'Setting up workspace...')
}
// We also need the jenkins-conf repository for the memory checking
// scripts.
sh '''
git clone https://github.com/mlpack/jenkins-conf
'''
@@ -44,6 +54,8 @@ pipeline
{
steps
{
script { u.updateCheckStatus('Building mlpack...') }
sh '''
export CCACHE_DIR=/opt/ccache/;
ccache --zero-stats
@@ -69,6 +81,8 @@ pipeline
// what files have changed.
script
{
u.updateCheckStatus('Running memory checks...')
if (env.BRANCH_NAME.startsWith('PR-'))
{
// Strip 'PR-' from the front.
@@ -112,9 +126,20 @@ pipeline
post
{
success
{
script { u.finishCheck('No memory errors found.', true) }
}
failure
{
script { u.finishCheck('Memory errors found.', false) }
}
always
{
junit(allowEmptyResults: true,
skipPublishingChecks: true,
testResults: '**/reports/tests/*.xml')
// Clean the workspace.
@@ -122,16 +147,6 @@ pipeline
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
// Set the build status.
script
{
u.setBuildStatus(result: currentBuild.currentResult,
context: "Memory Checks",
successMessage: "No memory errors.",
unstableMessage: "Build unstable.",
failureMessage: "Memory check failure with valgrind.");
}
}
}
}
.jenkins/style-checks/Jenkinsfile
Vendored
+20 -22
View File
@@ -16,20 +16,28 @@ pipeline
{
// Only allow one build at a time of this job.
disableConcurrentBuilds(abortPrevious: true)
// We will do checkout manually.
skipDefaultCheckout()
}
stages
{
// First we have to check out the jenkins-conf repository, which contains
// the scripts that we will use for checking the style.
stage('Check out jenkins-conf repository')
// Clean the workspace and check out the code.
stage('Set up workspace')
{
steps
{
cleanWs(deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
checkout scm
script
{
// Set the initial status.
u = load '.jenkins/utils.groovy'
u.startBuild('Style Checks')
u.startCheck('Style checks', 'Setting up workspace...')
}
sh '''
@@ -43,6 +51,8 @@ pipeline
{
steps
{
script { u.updateCheckStatus('Checking code style...') }
sh '''
mkdir -p reports
./jenkins-conf/linter/lint.sh \
@@ -60,34 +70,22 @@ pipeline
post
{
// Mark unstable builds as failed.
unstable
{
script
{
error "Style check failure."
}
}
unstable { script { u.finishCheck('Style checks failed.', false) } }
failure { script { u.finishCheck('Style checks failed.', false) } }
success { script { u.finishCheck('Style checks passed.', true) } }
always
{
// Process the test results.
junit(allowEmptyResults: true,
skipPublishingChecks: true,
testResults: '**/reports/cpplint.junit.xml')
// Clean the workspace.
cleanWs(cleanWhenNotBuilt: false,
// Clean the workspace after the build too.
cleanWs(cleanWhenNotBuilt: true,
deleteDirs: true,
disableDeferredWipeout: true,
notFailBuild: true)
script
{
u.setBuildStatus(result: currentBuild.currentResult,
context: "Style Checks",
successMessage: "No style issues.",
unstableMessage: "Style issues found.",
failureMessage: "Style issues found.");
}
}
}
}
.jenkins/utils.groovy
+43 -55
View File
@@ -1,66 +1,54 @@
// A simple utility to mark the build as pending on Github.
def startBuild(String context)
// These variables are set to hold state between status check updates.
def name = ''
def status = ''
def time = 0
def startCheck(String name, String status)
{
step([
$class: "GitHubCommitStatusSetter",
reposSource: [$class: "ManuallyEnteredRepositorySource",
url: "https://github.com/mlpack/mlpack"],
contextSource: [$class: "ManuallyEnteredCommitContextSource",
context: context ],
errorHandlers: [[$class: "ChangingBuildStatusErrorHandler",
result: "UNSTABLE"]],
statusResultSource: [$class: "ConditionalStatusResultSource",
results: [[$class: "AnyBuildResult",
message: "Building...",
state: "PENDING"]]]
]);
// Set module-level variables that we will retain as we build.
this.name = name
this.status = status
this.time = currentBuild.duration
publishChecks(name: name,
status: 'IN_PROGRESS',
title: status,
text: status,
detailsURL: currentBuild.absoluteUrl + 'console')
}
// A simple utility to set the build status on Github for a commit.
def setBuildStatus(Map paramsMap)
def updateCheckStatus(String status)
{
// Extract arguments from the map.
def result = paramsMap.result;
def context = paramsMap.context;
def successMessage = paramsMap.successMessage;
def unstableMessage = paramsMap.unstableMessage;
def failureMessage = paramsMap.failureMessage;
def stepTime = (currentBuild.duration - this.time) / 1000.0
this.status += ' (' + stepTime.toString() + 's)\n'
this.status += status
this.time = currentBuild.duration
def message = "(unknown Jenkins build result)";
def state = "FAILURE";
if (result == "FAILURE")
publishChecks(name: this.name,
status: 'IN_PROGRESS',
title: status,
text: this.status,
detailsURL: currentBuild.absoluteUrl + 'console')
}
def finishCheck(String status, boolean success)
{
def stepTime = (currentBuild.duration - this.time) / 1000.0
this.status += ' (' + stepTime.toString() + 's)\n'
this.status += status
if (!success)
{
message = failureMessage;
}
else if (result == "UNSTABLE")
{
message = unstableMessage;
state = "UNSTABLE";
}
else if (result == "SUCCESS")
{
message = successMessage;
state = "SUCCESS";
}
else if (result == "ABORTED")
{
message = "Job aborted.";
state = "ERROR";
this.status += '\n\n' +
'<b>Click \'view more details\' below to see failure details...</b>';
}
step([
$class: "GitHubCommitStatusSetter",
reposSource: [$class: "ManuallyEnteredRepositorySource",
url: "https://github.com/mlpack/mlpack"],
contextSource: [$class: "ManuallyEnteredCommitContextSource",
context: context ],
errorHandlers: [[$class: "ChangingBuildStatusErrorHandler",
result: "UNSTABLE"]],
statusResultSource: [$class: "ConditionalStatusResultSource",
results: [[$class: "AnyBuildResult",
message: message,
state: state]]]
]);
publishChecks(name: this.name,
status: 'COMPLETED',
conclusion: success ? 'SUCCESS' : 'FAILURE',
title: status,
text: this.status,
detailsURL: currentBuild.absoluteUrl + 'testReport')
}
return this