diff --git a/.github/workflows/jenkins.yaml b/.github/workflows/jenkins.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a9ab1fba7d822ef33a11556cdc17bcdba33dfa62
--- /dev/null
+++ b/.github/workflows/jenkins.yaml
@@ -0,0 +1,11 @@
+name: Trigger Jenkins Job
+on:
+ repository_dispatch:
+ types: [start-build]
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Trigger Jenkins Job
+ run: |
+ curl -X POST "http://jenkins.epic.oarcloud.noaa.gov/job/global-workflow/EMC-Pipelines/PR-274/buildWithParameters?token=${{ secrets.JENKINS_TOKEN }}&MACINE=${{ github.event.client_payload.param_value }}"
\ No newline at end of file
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index b1334e534450c78a688eb10a9326aab0b6de7f02..162d55ca92405ff8f8e9350b619065691a2a87e3 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -19,6 +19,7 @@ pipeline {
stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
// which is used to designate the Nodes in the Jenkins Controler by the agent label
// Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+ // no op 2
stage('Get Machine') {
agent { label 'built-in' }
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
new file mode 100644
index 0000000000000000000000000000000000000000..144493009c4beaf31a175bd7dfad1c5b0a050771
--- /dev/null
+++ b/ci/Jenkinsfile_multilabel
@@ -0,0 +1,311 @@
+def Machine = 'none'
+def machine = 'none'
+def HOME = 'none'
+def caseList = ''
+// no op 2 Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
+def repo_tmp = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
+def repo_url = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
+def STATUS = 'Passed'
+
+properties([
+ parameters([[
+ $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ string(name: 'machine', defaultValue: 'orion', description: 'Node to run on today'),
+ ])
+])
+
+pipeline {
+
+ agent { label 'built-in' }
+
+ options {
+ skipDefaultCheckout()
+ buildDiscarder(logRotator(numToKeepStr: '3'))
+ //parallelsAlwaysFailFast()
+ }
+
+ stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
+ // which is used to designate the Nodes in the Jenkins Controler by the agent label
+ // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+
+ stage('Get Machine') {
+ //when {
+ // //expression { env.BUILD_CAUSE != 'UPSTREAMTRIGGER' }
+ // expression { env.Node == 'built-in' }
+ //}
+ agent { label 'built-in' }
+ steps {
+ script {
+ // generateBaselineNode = []
+ // . for (label in pullRequest.labels) {
+ // listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
+ // node -> node.getLabelString().contains(label) ? node.name : null
+
+ // if ((label.matches(node.getLabelString()+"-(.*)"))) {
+ // matchedNode += node.getLabelString()
+ // }
+
+ // if ((label.matches(node.getLabelString()+"(.*)-emc"))) {
+ // generateBaselineNode += node.getLabelString()
+ // }
+ // }
+ // }
+ echo "Debugging Information: in Get Machine"
+ echo "====================================="
+ echo "machine: ${machine}"
+ echo "Node: ${env.Node}"
+ echo "Change ID: ${env.CHANGE_ID}"
+ echo "env.BUILD_CAUSE: ${env.BUILD_CAUSE}"
+
+ if ( env.Node != 'built-in' ) {
+ echo "machine being set to value passed to job"
+ echo "passed machine: ${params.machine}"
+ machine = params.machine
+ } else {
+ echo "Lanuching second build job on Hercules-EMC"
+ build job: "/global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [
+ string(name: 'machine', value: 'hercules'),
+ string(name: 'Node', value: 'Hercules-EMC') ],
+ wait: false
+ echo "Continuing with build on Hera-EMC"
+ machine = 'hera'
+ }
+ }
+ }
+ }
+
+ stage('Get Common Workspace') {
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+
+ echo "Debugging Information: in Get Common Workspace"
+ echo "====================================="
+ echo "machine: ${machine}"
+ echo "Node: ${env.Node}"
+ echo "Change ID: ${env.CHANGE_ID}"
+ echo "env.BUILD_CAUSE: ${env.BUILD_CAUSE}"
+
+ Machine = machine[0].toUpperCase() + machine.substring(1)
+ ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
+ HOME = "${WORKSPACE}"
+ sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
+ }
+ echo "Building and running on ${Machine} in directory ${HOME}"
+ }
+ }
+ }
+
+ stage('Build System') {
+ matrix {
+ agent { label "${machine}-emc" }
+ //options {
+ // throttle(['global_matrix_build'])
+ //}
+ axes {
+ axis {
+ name 'system'
+ values 'gfs', 'gefs'
+ }
+ }
+ stages {
+ stage('build system') {
+ steps {
+ script {
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
+ sh(script: "mkdir -p ${HOMEgfs}")
+ ws(HOMEgfs) {
+ if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
+ sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
+ checkout scm
+ dir('sorc') {
+ sh(script: './link_workflow.sh')
+ }
+ } else {
+ if (env.CHANGE_ID) {
+ dir("${HOMEgfs}") {
+ sh(script: "cd ${HOMEgfs}")
+ }
+ try {
+ echo "Checkout out PR-${env.CHANGE_ID} from ${repo_url}"
+ echo "To also be clone in directory: ${pwd()}"
+ checkout scm
+ // no op 2
+ //checkout([
+ // $class: 'GitSCM',
+ // userRemoteConfigs: [[url: "${repo_url}", credentialsId: '4ecd9d87-9253-45b0-929c-0dab37905d69']],
+ // branches: [[name: "refs/pull/${env.CHANGE_ID}/head"]],
+ //])
+ } catch (Exception e) {
+ echo "Failed to checkout scm: ${e.getMessage()}"
+ }
+ } else {
+ echo "Not a pull request, skipping checkout"
+ }
+ def gist_url = ""
+ def error_logs = ""
+ def error_logs_message = ""
+ def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
+ def build_args_list = builds_file['builds']
+ def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
+ dir("${HOMEgfs}/sorc") {
+ try {
+ sh(script: "${build_args}")
+ } catch (Exception error_build) {
+ echo "Failed to build system: ${error_build.getMessage()}"
+ if ( fileExists("logs/error.logs") ) {
+ def fileContent = readFile 'logs/error.logs'
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${line}") && readFile("${line}").length() > 0 ){
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
+ error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
+ }
+ catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
+ }
+ }
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+ gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ error("Failed to build system on ${Machine}")
+ }
+ }
+ sh(script: './link_workflow.sh')
+ sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ }
+ }
+ if (env.CHANGE_ID && system == 'gfs') {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
+ } catch (Exception e) {
+ echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+ }
+ }
+ if (system == 'gfs') {
+ caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ stage('Run Tests') {
+ failFast false
+ matrix {
+ agent { label "${machine}-emc" }
+ axes {
+ axis {
+ name 'Case'
+ // TODO add dynamic list of cases from env vars (needs addtional plugins)
+ values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
+ }
+ }
+ stages {
+
+ stage('Create Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ steps {
+ script {
+ sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
+ def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
+ system = yaml_case.experiment.system
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
+ env.RUNTESTS = "${HOME}/RUNTESTS"
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml")
+ }
+ }
+ }
+
+ stage('Run Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ failFast false
+ steps {
+ script {
+ HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
+ def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
+ } catch (Exception error_experment) {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
+ ws(HOME) {
+ def error_logs = ""
+ def error_logs_message = ""
+ def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
+ if (fileExists(error_file)) {
+ def fileContent = readFile error_file
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${HOME}/${line}") && readFile("${HOME}/${line}").length() > 0) {
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOME}/${line} "
+ error_logs_message = error_logs_message + "${HOME}/${line}\n"
+ } catch (Exception error_arch) {
+ echo "Failed to archive error log ${line}: ${error_arch.getMessage()}"
+ }
+ }
+ }
+ try {
+ gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ } else {
+ echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
+ }
+ STATUS = 'Failed'
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ error("Failed to run experiments ${Case} on ${Machine}")
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ stage( 'FINALIZE' ) {
+ when {
+ expression {
+ STATUS == 'Passed'
+ }
+ }
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ try {
+ def currentDate = new Date()
+ def date = currentDate.format('MMMM dd HH:mm')
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at ${date} Built and ran in directory<br>\\`${HOME}\\`" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ }
+ }
+ }
+ }
+}