From be21834d054adbfb4438ae5ae43f0c74fb04712b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 18:58:29 +0000 Subject: [PATCH 001/447] added basic Jenkinsfile --- Jenkinsfile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..b663b5297 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,22 @@ +pipeline { + agent DemoJNLP + + environment { + REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git' + PR_NUMBER = '203' + GITHUB_TOKEN = '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' + } + + + stage('Checkout') { + steps { + git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}" + } + } + + stage('Build') { + steps { + sh 'touch this_examle' + } + } + } -- GitLab From c969c8d6b936071496b8ccff54d169fa1c25e81c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 19:13:57 +0000 Subject: [PATCH 002/447] added node name to Jenkinsfile --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index b663b5297..488a08c5a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { GITHUB_TOKEN = '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' } + node ('DemoJNPL') { stage('Checkout') { steps { @@ -20,3 +21,5 @@ pipeline { } } } + + } -- GitLab From b94a115ca7db3ee0987ba64491b36b49aee067e3 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Thu, 7 Dec 2023 19:26:32 +0000 Subject: [PATCH 003/447] Update Jenkinsfile --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 488a08c5a..03c30339c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,6 @@ pipeline { - agent DemoJNLP + agent('DemoJNLP') { + environment { REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git' @@ -23,3 +24,4 @@ pipeline { } } +} -- GitLab From 79148ec51e8d2a7c2bc22becc356d7cc11f70096 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 19:46:01 +0000 Subject: [PATCH 004/447] updaet syntax on Jenkinsfile --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 03c30339c..52b9a8651 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,5 @@ pipeline { - agent('DemoJNLP') { - + agent( label 'demoJNLP') environment { REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git' @@ -10,7 +9,9 @@ pipeline { node ('DemoJNPL') { - stage('Checkout') { + stages { + + stage('Checkout') { steps { git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}" } @@ -23,5 +24,4 @@ pipeline { } } - } -} +} \ No newline at end of file -- GitLab From 8096cee74d07b6f8f492e5432b7d5c19c28ae908 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 19:53:39 +0000 Subject: [PATCH 005/447] brackets on agaent --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 52b9a8651..7b2495925 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent( label 'demoJNLP') + agent{ label 'demoJNLP'} environment { REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git' -- GitLab From 5cb82cdde47350d1fb13a8a8492f14e82866a366 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 19:56:53 +0000 Subject: [PATCH 006/447] missed bracket --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7b2495925..a43481ace 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,5 +23,5 @@ pipeline { } } } - + } } \ No newline at end of file -- GitLab From ed3e612af55c81b080c3098133d78c09792aa975 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 20:00:22 +0000 Subject: [PATCH 007/447] node goes in stages over each step --- Jenkinsfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a43481ace..b1010c02c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,14 +7,12 @@ pipeline { GITHUB_TOKEN = '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' } - node ('DemoJNPL') { - stages { - + node ('DemoJNPL') { stage('Checkout') { - steps { + steps { git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}" - } + } } stage('Build') { -- GitLab From 8f1baeb4eac9c60d46c71916e8e5911ed0853d51 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 7 Dec 2023 20:05:25 +0000 Subject: [PATCH 008/447] took node out for now --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b1010c02c..311792f27 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,6 @@ pipeline { } stages { - node ('DemoJNPL') { stage('Checkout') { steps { git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}" @@ -21,5 +20,4 @@ pipeline { } } } - } } \ No newline at end of file -- GitLab From 95d209d27e2cee459f0163e0f22d00aa470ac31d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 20:49:39 +0000 Subject: [PATCH 009/447] do a push --- ci/scripts/noop | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ci/scripts/noop diff --git a/ci/scripts/noop b/ci/scripts/noop new file mode 100644 index 000000000..e69de29bb -- GitLab From bbdbf7b9daca54b614bd275856b53d7f971cf8e3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 21:14:42 +0000 Subject: [PATCH 010/447] checkout PR --- Jenkinsfile | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 311792f27..f3d86c7a5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,23 +1,12 @@ pipeline { agent{ label 'demoJNLP'} - environment { - REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git' - PR_NUMBER = '203' - GITHUB_TOKEN = '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' - } - stages { - stage('Checkout') { - steps { - git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}" - } - } - - stage('Build') { + stage('Checkout Repos') { steps { - sh 'touch this_examle' + sh './source/checkout.sh -c -g -u' } + } } } \ No newline at end of file -- GitLab From 52ae1aa72130e17cf4bb24404797809f914ca1c0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 21:22:50 +0000 Subject: [PATCH 011/447] updated path to checkout.sh --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f3d86c7a5..266d34344 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,9 +4,9 @@ pipeline { stages { stage('Checkout Repos') { steps { - sh './source/checkout.sh -c -g -u' + sh './sorc/checkout.sh -c -g -u' } } } -} \ No newline at end of file +} -- GitLab From 7e043e3d326a4c5e16a31bb7877401c4a411bcd8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 21:51:32 +0000 Subject: [PATCH 012/447] add label --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 266d34344..7d4d6517a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ pipeline { steps { sh './sorc/checkout.sh -c -g -u' } - + githubPRAddLabels labelProperty: labels('CI-Orion-Built') } } } -- GitLab From 8b23d6e9bd833a53116f43464392896d2b827249 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 22:04:41 +0000 Subject: [PATCH 013/447] add label --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d4d6517a..e2a298090 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,9 +4,9 @@ pipeline { stages { stage('Checkout Repos') { steps { - sh './sorc/checkout.sh -c -g -u' + sh './sorc/checkout.sh -c -g -u' + pullRequest.addLabel('CI-Orion-Passed') } - githubPRAddLabels labelProperty: labels('CI-Orion-Built') } } } -- GitLab From 10986b48b35f4f6f78bc80dfa394d74384759e53 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 22:14:22 +0000 Subject: [PATCH 014/447] noop change --- ci/scripts/noop | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ci/scripts/noop diff --git a/ci/scripts/noop b/ci/scripts/noop deleted file mode 100644 index e69de29bb..000000000 -- GitLab From a4ccc5e3fa58b3d82a6f67f1d79fb6903e6dc6ad Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 22:16:17 +0000 Subject: [PATCH 015/447] moving add Label around --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e2a298090..28351ebe8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,8 +5,8 @@ pipeline { stage('Checkout Repos') { steps { sh './sorc/checkout.sh -c -g -u' - pullRequest.addLabel('CI-Orion-Passed') } + pullRequest.addLabel('CI-Orion-Passed') } } } -- GitLab From 907d31aa09ce5670d6fe0df291df23e525402048 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 8 Dec 2023 22:24:57 +0000 Subject: [PATCH 016/447] added labels add in post SUCCESS --- Jenkinsfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28351ebe8..69062908f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,12 +1,18 @@ pipeline { - agent{ label 'demoJNLP'} + agent{ label 'demoJNLP'} stages { stage('Checkout Repos') { steps { - sh './sorc/checkout.sh -c -g -u' + sh './sorc/checkout.sh -c -g -u' } - pullRequest.addLabel('CI-Orion-Passed') } } -} + post { + success { + script { + pullRequest.labels.add('CI-Orion-Passed') + } + } + } +} \ No newline at end of file -- GitLab From 29de307023159567512a5f556ac7320c3739c29e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 11 Dec 2023 17:30:38 +0000 Subject: [PATCH 017/447] added correct credntialsId for label add --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 69062908f..bf60bfdda 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,7 @@ pipeline { post { success { script { + withCredentials([string(credentialsId: '7dcc5bf3-cc6a-48b1-acd8-c1dc35e3b10b', variable: 'GITHUB_TOKEN')]) pullRequest.labels.add('CI-Orion-Passed') } } -- GitLab From 772f8b84285cdae7f5f96f950575f8a9fb07b87b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 11 Dec 2023 18:20:59 +0000 Subject: [PATCH 018/447] removed credentials block --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bf60bfdda..69062908f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,6 @@ pipeline { post { success { script { - withCredentials([string(credentialsId: '7dcc5bf3-cc6a-48b1-acd8-c1dc35e3b10b', variable: 'GITHUB_TOKEN')]) pullRequest.labels.add('CI-Orion-Passed') } } -- GitLab From b6568de7e83ee605eecf5145df8fed2c8d6b6c8a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 11 Dec 2023 21:10:07 +0000 Subject: [PATCH 019/447] updated Node name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 69062908f..cf58cb432 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent{ label 'demoJNLP'} + agent{ label 'Sanbox'} stages { stage('Checkout Repos') { -- GitLab From e09e1dd6caa4e3c6dbe4d49097ef8a8bee95c1f8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 14 Dec 2023 20:58:40 +0000 Subject: [PATCH 020/447] basic sm and configured to include submodules --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cf58cb432..30f632841 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,11 +1,10 @@ pipeline { - agent{ label 'Sanbox'} + agent{ label 'local'} stages { - stage('Checkout Repos') { - steps { - sh './sorc/checkout.sh -c -g -u' - } + stage('Checkout') { + node{ + checkout scm } } post { @@ -15,4 +14,4 @@ pipeline { } } } -} \ No newline at end of file +} -- GitLab From 950ca196968429386ac60b4fad70e81bcb7b0331 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 14 Dec 2023 21:13:17 +0000 Subject: [PATCH 021/447] missed a bracket in Jenkinsfile --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30f632841..3f055a759 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,8 +3,9 @@ pipeline { stages { stage('Checkout') { - node{ - checkout scm + node{ + checkout scm + } } } post { -- GitLab From c23e8bda3715bf8ea7d6cb907689782861d60b9b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 14 Dec 2023 21:26:31 +0000 Subject: [PATCH 022/447] repalced node with step becuse first block in stage is steps --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3f055a759..89e3c239d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('Checkout') { - node{ + steps { checkout scm } } -- GitLab From c3bf6b2ef63dc26e4543dc9744fd559ee5635b55 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 14 Dec 2023 22:17:19 +0000 Subject: [PATCH 023/447] updated methond addLabel for pullResest --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 89e3c239d..aef4f1c2e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { post { success { script { - pullRequest.labels.add('CI-Orion-Passed') + pullRequest.addLabel(['CI-Orion-Passed']) } } } -- GitLab From 79b7bdcb3486465254d024a38378a8aea104c65a Mon Sep 17 00:00:00 2001 From: Terry McGuinness <terry.mcguinness@noaa.gov> Date: Tue, 19 Dec 2023 13:07:54 -0600 Subject: [PATCH 024/447] added label to node to match that on EPICs controler orion-emc --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index aef4f1c2e..254120396 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent{ label 'local'} + agent{ label 'orion-emc'} stages { stage('Checkout') { -- GitLab From 9380545e36125e6922b41406bcc40432374c5d82 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 20:58:39 +0000 Subject: [PATCH 025/447] added simple full checkout and FF build and the C48_ATM experment dir --- Jenkinsfile | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 254120396..7a9af735a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,11 +3,32 @@ pipeline { stages { stage('Checkout') { - steps { + steps { checkout scm + sh 'git submodule update --init --recursive' + } + } + stage('Build') { + steps { + sh 'sorc/build_all.sh' + } + } + stage ('Create Experment') { + steps { + sh ''' + export HOMEgfs=$PWD + mkdir -p RUNTESTS + export RUNTESTS=$PWD/RUNTESTS + source ci/platforms/config.orion.sh + pr_sha=$(git rev-parse --short HEAD) + case=C48_ATM + export pslot=${case}_${pr_sha} + workflow/create_experment.py --yaml ci/cases/pr/${case}' + ''' } } } + post { success { script { -- GitLab From 4b41cdc2cc6a92fb2caf1cd431af01b7b6d39cb5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 21:26:33 +0000 Subject: [PATCH 026/447] added link as a build step, fixed typo with cnofig.orion call --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7a9af735a..bdbac19f7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,7 @@ pipeline { stage('Build') { steps { sh 'sorc/build_all.sh' + sh 'sourc/link_workflow.sh' } } stage ('Create Experment') { @@ -19,7 +20,7 @@ pipeline { export HOMEgfs=$PWD mkdir -p RUNTESTS export RUNTESTS=$PWD/RUNTESTS - source ci/platforms/config.orion.sh + source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} -- GitLab From 10263d59138e183964d5bdf77ccf61a4f495c1a7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 21:48:45 +0000 Subject: [PATCH 027/447] corrected pash to global link script ins sorc --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bdbac19f7..2bf356aec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { stage('Build') { steps { sh 'sorc/build_all.sh' - sh 'sourc/link_workflow.sh' + sh 'sorc/link_workflow.sh' } } stage ('Create Experment') { -- GitLab From fb3b4a192bccf31f230a399f20394d5ca37c4e4f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 22:24:49 +0000 Subject: [PATCH 028/447] hanging quote in sh for creating an experment --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2bf356aec..d770d4b94 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} - workflow/create_experment.py --yaml ci/cases/pr/${case}' + workflow/create_experment.py --yaml ci/cases/pr/${case} ''' } } -- GitLab From 8970be928ff7cf5756729b5ebbabafa59e294cac Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 23:09:40 +0000 Subject: [PATCH 029/447] missed i in experiment in Jenkinsfile --- Jenkinsfile | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d770d4b94..af0ad1cbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,6 +5,8 @@ pipeline { stage('Checkout') { steps { checkout scm + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') sh 'git submodule update --init --recursive' } } @@ -16,16 +18,19 @@ pipeline { } stage ('Create Experment') { steps { - sh ''' - export HOMEgfs=$PWD - mkdir -p RUNTESTS - export RUNTESTS=$PWD/RUNTESTS - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - case=C48_ATM - export pslot=${case}_${pr_sha} - workflow/create_experment.py --yaml ci/cases/pr/${case} - ''' + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + sh ''' + export HOMEgfs=$PWD + mkdir -p RUNTESTS + export RUNTESTS=$PWD/RUNTESTS + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + case=C48_ATM + export pslot=${case}_${pr_sha} + workflow/create_experiment.py --yaml ci/cases/pr/${case} + ''' + pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') } } } @@ -33,7 +38,8 @@ pipeline { post { success { script { - pullRequest.addLabel(['CI-Orion-Passed']) + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Passed') } } } -- GitLab From 14b722e77346b366895c62894bdace3dd894aeee Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 23:29:59 +0000 Subject: [PATCH 030/447] added scripts around pullRequest methods in Jenkinsfile --- Jenkinsfile | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index af0ad1cbe..0e5e73511 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,8 +5,10 @@ pipeline { stage('Checkout') { steps { checkout scm - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') + script { + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') + } sh 'git submodule update --init --recursive' } } @@ -18,8 +20,10 @@ pipeline { } stage ('Create Experment') { steps { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + } sh ''' export HOMEgfs=$PWD mkdir -p RUNTESTS @@ -28,9 +32,11 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} - workflow/create_experiment.py --yaml ci/cases/pr/${case} + workflow/create_experment.py --yaml ci/cases/pr/${case} ''' - pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') + script { + pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') + } } } } @@ -42,5 +48,11 @@ pipeline { pullRequest.addLabel('CI-Orion-Passed') } } + failure { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Failed') + } + } } } -- GitLab From 4564388f522ba529835175d3a83d78291022b987 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 20 Dec 2023 23:55:02 +0000 Subject: [PATCH 031/447] missed update to experiment in create experiment script in Jenkins file (another redu) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0e5e73511..9694723de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} - workflow/create_experment.py --yaml ci/cases/pr/${case} + workflow/create_experiment.py --yaml ci/cases/pr/${case} ''' script { pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') -- GitLab From 03de881532776321efd0b53bae376bded841ce50 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 20 Dec 2023 20:33:23 -0500 Subject: [PATCH 032/447] Update Jenkinsfile --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 9694723de..67d21dfcc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,6 +32,7 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} + source workflow/gwsetup.sh workflow/create_experiment.py --yaml ci/cases/pr/${case} ''' script { -- GitLab From 6849a1646890d2d2fbbedc53db5a277636d9cd13 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 21 Dec 2023 01:55:54 +0000 Subject: [PATCH 033/447] misspelled another path ... underscore in gw setup --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 67d21dfcc..128738ab0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) case=C48_ATM export pslot=${case}_${pr_sha} - source workflow/gwsetup.sh + source workflow/gw_setup.sh workflow/create_experiment.py --yaml ci/cases/pr/${case} ''' script { -- GitLab From bf49b89ef996494a41d9d87e00ff6f90b8946d33 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 21 Dec 2023 02:20:58 +0000 Subject: [PATCH 034/447] need extention yaml on case name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 128738ab0..1b2b33e4c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,7 @@ pipeline { case=C48_ATM export pslot=${case}_${pr_sha} source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case} + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml ''' script { pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') -- GitLab From b4792992fbceb399356d42d1a0ac9295dff42088 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 16:42:40 +0000 Subject: [PATCH 035/447] renaming Jenkinsfile inorder to remove passed run logs on controler --- Jenkinsfile => Jenkinsfile_temp | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile => Jenkinsfile_temp (100%) diff --git a/Jenkinsfile b/Jenkinsfile_temp similarity index 100% rename from Jenkinsfile rename to Jenkinsfile_temp -- GitLab From 38c51cdc238ab502d5c028f519049092d4742d8f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 16:44:21 +0000 Subject: [PATCH 036/447] rename Jenkinsfile back to org to start fresh pipline --- Jenkinsfile_temp => Jenkinsfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile_temp => Jenkinsfile (100%) diff --git a/Jenkinsfile_temp b/Jenkinsfile similarity index 100% rename from Jenkinsfile_temp rename to Jenkinsfile -- GitLab From 9557169820d2391c0f3c567d322a22624559715d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 19:01:37 +0000 Subject: [PATCH 037/447] generlized stages for creating each pr case in pipeline --- Jenkinsfile | 46 ++++++++++++++++------------ ci/scripts/utils/ci_utils.sh | 13 ++++++++ ci/scripts/utils/ci_utils_wrapper.sh | 6 ++++ 3 files changed, 46 insertions(+), 19 deletions(-) create mode 100755 ci/scripts/utils/ci_utils_wrapper.sh diff --git a/Jenkinsfile b/Jenkinsfile index 1b2b33e4c..52f10d1de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,11 @@ pipeline { agent{ label 'orion-emc'} + environment { + HOMEgfs = "${WORKSPACE}" + RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + stages { stage('Checkout') { steps { @@ -16,30 +21,33 @@ pipeline { steps { sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' + env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() } } - stage ('Create Experment') { - steps { - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - } - sh ''' - export HOMEgfs=$PWD - mkdir -p RUNTESTS - export RUNTESTS=$PWD/RUNTESTS - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - case=C48_ATM - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml - ''' - script { - pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') + + def cases = env.CASE_LIST.tokenize('\n') + + cases.each { case_name -> + stage("Run ${case_name}") { + steps { + sh ''' + export HOMEgfs=${env.HOMEgfs} + mkdir -p ${env.RUNTESTS} + export RUNTESTS=${env.RUNTESTS} + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + case=${case_name} + export pslot=${case}_${pr_sha} + source workflow/gw_setup.sh + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml + ''' + script { + pullRequest.comment("SUCCESS creating ${case_name} on Orion") + } } } } + } post { diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 737a3e5a8..cc9f8aaa2 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -22,3 +22,16 @@ function cancel_slurm_jobs() { fi done } + +function get_pr_case_list () { + + ############################################################# + # loop over every yaml file in the PR's ci/cases + # and create an run directory for each one for this PR loop + ############################################################# + echo ${HOMEgfs} + for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do + case=$(basename "${yaml_config}" .yaml) || true + echo "${case}" + done +} \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh new file mode 100755 index 000000000..f3cd461e8 --- /dev/null +++ b/ci/scripts/utils/ci_utils_wrapper.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +utitilty_function="${1}" + +source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" +${utitilty_function} "${@:2}" \ No newline at end of file -- GitLab From d959942da3fb60935f204d9227983be5d8d90843 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 19:10:59 +0000 Subject: [PATCH 038/447] Jenkinsfile syntax step over env def --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 52f10d1de..74ddceb56 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,8 @@ pipeline { steps { sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' + } + steps { env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() } } -- GitLab From afb3789f412569218928b331a66d1b4fbb54a392 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 19:46:36 +0000 Subject: [PATCH 039/447] simply made cases pipeline global --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 74ddceb56..cf541e973 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,5 @@ + +def cases="cases" pipeline { agent{ label 'orion-emc'} @@ -21,14 +23,13 @@ pipeline { steps { sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' - } - steps { - env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + script { + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases=case_list.tokenize('\n') + } } } - def cases = env.CASE_LIST.tokenize('\n') - cases.each { case_name -> stage("Run ${case_name}") { steps { -- GitLab From b7a7a6042714b6256095705a4c1397643e0093f2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 19:57:57 +0000 Subject: [PATCH 040/447] adding next of cases as a new stage --- Jenkinsfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cf541e973..9a3d44b11 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,9 +29,11 @@ pipeline { } } } - - cases.each { case_name -> - stage("Run ${case_name}") { + + stage('Create Cases') { + stages { + cases.each { case_name -> + stage("Run ${case_name}") { steps { sh ''' export HOMEgfs=${env.HOMEgfs} @@ -48,9 +50,10 @@ pipeline { pullRequest.comment("SUCCESS creating ${case_name} on Orion") } } + } + } } - } post { -- GitLab From 98068be1a084163aeea3c87c2e8b09351346a7ec Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 20:02:05 +0000 Subject: [PATCH 041/447] missed a bracket in new nest of cases --- Jenkinsfile | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9a3d44b11..e9954624e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,28 +31,29 @@ pipeline { } stage('Create Cases') { - stages { - cases.each { case_name -> - stage("Run ${case_name}") { - steps { - sh ''' - export HOMEgfs=${env.HOMEgfs} - mkdir -p ${env.RUNTESTS} - export RUNTESTS=${env.RUNTESTS} - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - case=${case_name} - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml - ''' - script { - pullRequest.comment("SUCCESS creating ${case_name} on Orion") + stages { + cases.each { case_name -> + stage("Run ${case_name}") { + steps { + sh ''' + export HOMEgfs=${env.HOMEgfs} + mkdir -p ${env.RUNTESTS} + export RUNTESTS=${env.RUNTESTS} + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + case=${case_name} + export pslot=${case}_${pr_sha} + source workflow/gw_setup.sh + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml + ''' + script { + pullRequest.comment("SUCCESS creating ${case_name} on Orion") + } + } + } } - } - } } } -- GitLab From 6b478014d3eda26aabffc047198cc469e771ddef Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 20:22:34 +0000 Subject: [PATCH 042/447] dynamic stages as steps script --- Jenkinsfile | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e9954624e..9fb0b6f4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,30 +31,32 @@ pipeline { } stage('Create Cases') { - stages { - cases.each { case_name -> - stage("Run ${case_name}") { - steps { - sh ''' - export HOMEgfs=${env.HOMEgfs} - mkdir -p ${env.RUNTESTS} - export RUNTESTS=${env.RUNTESTS} - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - case=${case_name} - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml - ''' - script { - pullRequest.comment("SUCCESS creating ${case_name} on Orion") - } + steps { + script { + cases.each { case_name -> + stage("Run ${case_name}") { + steps { + sh ''' + export HOMEgfs=${env.HOMEgfs} + mkdir -p ${env.RUNTESTS} + export RUNTESTS=${env.RUNTESTS} + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + case=${case_name} + export pslot=${case}_${pr_sha} + source workflow/gw_setup.sh + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml + ''' + script { + pullRequest.comment("SUCCESS creating ${case_name} on Orion") + } + } } - } } } - } + } + } post { -- GitLab From 547067ce76394bc8c3f6c6b220d5d71528ede5a0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 20:33:21 +0000 Subject: [PATCH 043/447] rename Jenkinsfile to clear --- Jenkinsfile => Jenkinsfile_org | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile => Jenkinsfile_org (100%) diff --git a/Jenkinsfile b/Jenkinsfile_org similarity index 100% rename from Jenkinsfile rename to Jenkinsfile_org -- GitLab From 3624df74ac56d3bdcb289025264aa7cfc65b408d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 20:35:34 +0000 Subject: [PATCH 044/447] rename Jenkinsfile to clear --- Jenkinsfile_org => Jenkinsfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile_org => Jenkinsfile (100%) diff --git a/Jenkinsfile_org b/Jenkinsfile similarity index 100% rename from Jenkinsfile_org rename to Jenkinsfile -- GitLab From 750b67f517603b384ca9b0aa6a8b3ba151a270e4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:16:53 +0000 Subject: [PATCH 045/447] check case liset --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9fb0b6f4f..87720432c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,12 +21,13 @@ pipeline { } stage('Build') { steps { - sh 'sorc/build_all.sh' + // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() cases=case_list.tokenize('\n') } + echo "cases: ${cases}" } } -- GitLab From 5edb4b17cb11a774b73b5958630c8e0beebd7360 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:18:31 +0000 Subject: [PATCH 046/447] check case liset --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 87720432c..3406f560e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,6 +34,8 @@ pipeline { stage('Create Cases') { steps { script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') cases.each { case_name -> stage("Run ${case_name}") { steps { -- GitLab From 04bf11e4f1c3c07b6bb89aaa924f2e0b1d55fedc Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:26:20 +0000 Subject: [PATCH 047/447] check case list in next stage and remove first element --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3406f560e..d35123590 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,9 @@ pipeline { steps { script { pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Running') + case.removeAt(0) + echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { steps { -- GitLab From e7e3a7e31fe98b6cf626b9685094fb032cde9e01 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:28:44 +0000 Subject: [PATCH 048/447] mispelled cases as case --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d35123590..d29c77415 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ pipeline { script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') - case.removeAt(0) + cases.removeAt(0) echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { -- GitLab From b00a597d1574bea543e61166efcbc2d622eb51e1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:35:41 +0000 Subject: [PATCH 049/447] remove first element before making global for cases list --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d29c77415..10eaa8cff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,6 +25,7 @@ pipeline { sh 'sorc/link_workflow.sh' script { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + case_list.removeAt(0) cases=case_list.tokenize('\n') } echo "cases: ${cases}" @@ -36,7 +37,6 @@ pipeline { script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') - cases.removeAt(0) echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { -- GitLab From a914b85cf30449ca0ffdd3c8042528fc18d4ff9d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:41:16 +0000 Subject: [PATCH 050/447] remove firs element locally after tokenize case list --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 10eaa8cff..fe0ea6966 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,10 +25,11 @@ pipeline { sh 'sorc/link_workflow.sh' script { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + case_list.tokenize('\n') case_list.removeAt(0) - cases=case_list.tokenize('\n') + cases=case_list + echo "cases: ${cases}" } - echo "cases: ${cases}" } } -- GitLab From e1f33f76676320266b6e579066e5c6bfa5c0c460 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:44:51 +0000 Subject: [PATCH 051/447] create seperate ouput string then tokenized it before removing first elmenent --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe0ea6966..3fc0dcf0b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,8 +24,8 @@ pipeline { // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - case_list.tokenize('\n') + case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + case_list = case_list_ouput.tokenize('\n') case_list.removeAt(0) cases=case_list echo "cases: ${cases}" -- GitLab From 489f198a007eef4ab6b016988c570ccce30df860 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:50:29 +0000 Subject: [PATCH 052/447] workaround admin permissions to use removeAt --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3fc0dcf0b..336f3fcf6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,6 @@ def cases="cases" +def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" pipeline { agent{ label 'orion-emc'} @@ -24,9 +25,9 @@ pipeline { // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { - case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() case_list = case_list_ouput.tokenize('\n') - case_list.removeAt(0) + //case_list.removeAt(0) cases=case_list echo "cases: ${cases}" } -- GitLab From 3f7312a5ad549cff5c4e1bf419812a1588f24f17 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:51:02 +0000 Subject: [PATCH 053/447] workaround admin permissions to use removeAt --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 336f3fcf6..1811703a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() case_list = case_list_ouput.tokenize('\n') //case_list.removeAt(0) + case_list.remove(0) cases=case_list echo "cases: ${cases}" } -- GitLab From c6d508c332c9501bd1224e8bfe10031689530bf8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:54:24 +0000 Subject: [PATCH 054/447] misspelled case list outout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1811703a6..73c45d401 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,7 +26,7 @@ pipeline { sh 'sorc/link_workflow.sh' script { //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - case_list = case_list_ouput.tokenize('\n') + case_list = case_list_output.tokenize('\n') //case_list.removeAt(0) case_list.remove(0) cases=case_list -- GitLab From be720eb82fea3038a45e0638d5df71bc26160d2f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 21:56:07 +0000 Subject: [PATCH 055/447] spaces on workaround --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 73c45d401..32e6e5c8b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,7 +26,7 @@ pipeline { sh 'sorc/link_workflow.sh' script { //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - case_list = case_list_output.tokenize('\n') + case_list = case_list_output.tokenize(' ') //case_list.removeAt(0) case_list.remove(0) cases=case_list -- GitLab From 8925066b33b8f22ab2c5e0fd96136fa1d1ffacd7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:10:00 +0000 Subject: [PATCH 056/447] took out steps in create cases stage because its scripited --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 32e6e5c8b..6816797bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,6 @@ pipeline { echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { - steps { sh ''' export HOMEgfs=${env.HOMEgfs} mkdir -p ${env.RUNTESTS} @@ -58,7 +57,6 @@ pipeline { script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") } - } } } } -- GitLab From 8abc5d058ec6ce22021ec6b5f5bb5273e4f7e7f9 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:10:47 +0000 Subject: [PATCH 057/447] took out steps in create cases stage because its scripited --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6816797bb..39f654872 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ def cases="cases" -def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" +//def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" pipeline { agent{ label 'orion-emc'} @@ -25,9 +25,8 @@ pipeline { // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { - //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() case_list = case_list_output.tokenize(' ') - //case_list.removeAt(0) case_list.remove(0) cases=case_list echo "cases: ${cases}" -- GitLab From e932f4fcbbc5458d7b9044b36417d9c8c24df6bb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:13:57 +0000 Subject: [PATCH 058/447] more added back get list with remove --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39f654872..bed73123e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,8 +25,8 @@ pipeline { // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { - case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - case_list = case_list_output.tokenize(' ') + case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + case_list = case_list_output.tokenize('\n') case_list.remove(0) cases=case_list echo "cases: ${cases}" -- GitLab From c39c6fa33a3199b1f9f59084a6d0f90fb444ba6d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:26:10 +0000 Subject: [PATCH 059/447] same agent accross all steps --- Jenkinsfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bed73123e..6ff644109 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ def cases="cases" //def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" pipeline { - agent{ label 'orion-emc'} + agent none environment { HOMEgfs = "${WORKSPACE}" @@ -11,6 +11,7 @@ pipeline { stages { stage('Checkout') { + agent{ label 'orion-emc'} steps { checkout scm script { @@ -21,6 +22,7 @@ pipeline { } } stage('Build') { + agent{ label 'orion-emc'} steps { // sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' @@ -35,6 +37,7 @@ pipeline { } stage('Create Cases') { + agent{ label 'orion-emc'} steps { script { pullRequest.removeLabel('CI-Orion-Building') @@ -42,6 +45,8 @@ pipeline { echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { + agent{ label 'orion-emc'} + steps { sh ''' export HOMEgfs=${env.HOMEgfs} mkdir -p ${env.RUNTESTS} @@ -56,6 +61,7 @@ pipeline { script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") } + } } } } @@ -65,6 +71,7 @@ pipeline { } post { + agent{ label 'orion-emc'} success { script { pullRequest.removeLabel('CI-Orion-Running') @@ -78,4 +85,4 @@ pipeline { } } } -} +} \ No newline at end of file -- GitLab From b74fd5856bd418436ff14f0c45aa0e3d45fc4b67 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:28:08 +0000 Subject: [PATCH 060/447] no agent in post --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6ff644109..47854dc45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -71,7 +71,6 @@ pipeline { } post { - agent{ label 'orion-emc'} success { script { pullRequest.removeLabel('CI-Orion-Running') -- GitLab From a29054115e3feca9c8dceb078648e1d9bd1d8f34 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:33:15 +0000 Subject: [PATCH 061/447] global parent stage on same agent --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47854dc45..415689b3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ def cases="cases" //def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" pipeline { - agent none + agent{ label 'orion-emc'} environment { HOMEgfs = "${WORKSPACE}" -- GitLab From 0a582c3e0a092dc68c25de45e66e5a88d9957eaa Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 22:43:08 +0000 Subject: [PATCH 062/447] take out steps in inner staged loop --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 415689b3a..7d9c3737f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { cases.each { case_name -> stage("Run ${case_name}") { agent{ label 'orion-emc'} - steps { + //steps { sh ''' export HOMEgfs=${env.HOMEgfs} mkdir -p ${env.RUNTESTS} @@ -61,7 +61,7 @@ pipeline { script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") } - } + //} } } } -- GitLab From f790d0feb37bb7c08989b3cea1c439fadc7f7795 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 23:13:50 +0000 Subject: [PATCH 063/447] HOMEgfs may already be assigined --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d9c3737f..a19cdbd43 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,9 +48,7 @@ pipeline { agent{ label 'orion-emc'} //steps { sh ''' - export HOMEgfs=${env.HOMEgfs} mkdir -p ${env.RUNTESTS} - export RUNTESTS=${env.RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) case=${case_name} -- GitLab From fe57fcc1eb69dea9083d0231f796a2e3616f2e0e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 23:20:47 +0000 Subject: [PATCH 064/447] still had env for RUNTESTS, removed it too --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a19cdbd43..b5407fd2a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,5 @@ def cases="cases" -//def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM" pipeline { agent{ label 'orion-emc'} @@ -48,7 +47,7 @@ pipeline { agent{ label 'orion-emc'} //steps { sh ''' - mkdir -p ${env.RUNTESTS} + mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) case=${case_name} -- GitLab From 8f0b5f6fb6f34611485d94ca37d0f3f10e3c1c8a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 23:43:59 +0000 Subject: [PATCH 065/447] removed case and just used case_name --- Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5407fd2a..5e8ee45cd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,13 +50,12 @@ pipeline { mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) - case=${case_name} - export pslot=${case}_${pr_sha} + export pslot=${case_name}_${pr_sha} source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml + workflow/create_experiment.py --yaml ci/cases/pr/${case_name}.yaml ''' script { - pullRequest.comment("SUCCESS creating ${case_name} on Orion") + pullRequest.comment("SUCCESS creating ${case_name} on Orion") } //} } -- GitLab From 59c4cecd6d35263f51f2b294d5f3fbbb666402e9 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 23:49:55 +0000 Subject: [PATCH 066/447] take out agent in sub stage for Run case --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e8ee45cd..ecb49e272 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,6 @@ pipeline { echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { - agent{ label 'orion-emc'} //steps { sh ''' mkdir -p ${RUNTESTS} -- GitLab From aa66559567f671b59751dbdd4de011dd4da7152e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 27 Dec 2023 23:54:00 +0000 Subject: [PATCH 067/447] case name is not in sub stage env --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ecb49e272..18b243144 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,12 +46,13 @@ pipeline { stage("Run ${case_name}") { //steps { sh ''' + case=${env.case_name} mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) - export pslot=${case_name}_${pr_sha} + export pslot=${case}_${pr_sha} source workflow/gw_setup.sh - workflow/create_experiment.py --yaml ci/cases/pr/${case_name}.yaml + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml ''' script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") -- GitLab From 2347e37c8aa682b5bdf7ef30f38373e716122723 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 00:02:59 +0000 Subject: [PATCH 068/447] still can not get case name in shell --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 18b243144..91e555d29 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,9 +44,10 @@ pipeline { echo "cases: ${cases}" cases.each { case_name -> stage("Run ${case_name}") { - //steps { + script { + env.case = case_name + } sh ''' - case=${env.case_name} mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) @@ -57,7 +58,6 @@ pipeline { script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") } - //} } } } -- GitLab From 364f2d36e6ef9d1644754ee2ab02de3127ac2c92 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 00:14:46 +0000 Subject: [PATCH 069/447] HOMEgfs gets NONE in xml so unset and local set --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 91e555d29..fe8f650be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,6 +53,8 @@ pipeline { pr_sha=$(git rev-parse --short HEAD) export pslot=${case}_${pr_sha} source workflow/gw_setup.sh + unset HOMEgfs + HOMEgfs=${WORKSPACE} workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml ''' script { -- GitLab From 8722098e22c7ce7bd917009467a1b312500d6e4b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 00:22:11 +0000 Subject: [PATCH 070/447] remove case dirs --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index fe8f650be..b2367a65f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,6 +48,7 @@ pipeline { env.case = case_name } sh ''' + rm -rf ${RUNTESTS} mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) -- GitLab From f6bb57e3e870460598d4e7aba72846e1a6e83645 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 00:32:08 +0000 Subject: [PATCH 071/447] rename Jenkinsfile to clear controler logs --- Jenkinsfile => Jenkinsfile_org | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile => Jenkinsfile_org (100%) diff --git a/Jenkinsfile b/Jenkinsfile_org similarity index 100% rename from Jenkinsfile rename to Jenkinsfile_org -- GitLab From b993492be1649bc7536344b00c8cfd2e3c94e37d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 00:34:33 +0000 Subject: [PATCH 072/447] rename stage Create Experiments --- Jenkinsfile_org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_org b/Jenkinsfile_org index b2367a65f..6bdf44dfa 100644 --- a/Jenkinsfile_org +++ b/Jenkinsfile_org @@ -23,7 +23,7 @@ pipeline { stage('Build') { agent{ label 'orion-emc'} steps { - // sh 'sorc/build_all.sh' + sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() @@ -43,7 +43,7 @@ pipeline { pullRequest.addLabel('CI-Orion-Running') echo "cases: ${cases}" cases.each { case_name -> - stage("Run ${case_name}") { + stage("Create ${case_name}") { script { env.case = case_name } -- GitLab From e93e0ae02d27f544381d8ef749e5365b60b7ff4d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 27 Dec 2023 21:21:23 -0500 Subject: [PATCH 073/447] Update Jenkinsfile_org moved RUNTESTS dir management out of loop --- Jenkinsfile_org | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile_org b/Jenkinsfile_org index 6bdf44dfa..93d168820 100644 --- a/Jenkinsfile_org +++ b/Jenkinsfile_org @@ -4,11 +4,11 @@ pipeline { agent{ label 'orion-emc'} environment { - HOMEgfs = "${WORKSPACE}" RUNTESTS = "${WORKSPACE}/RUNTESTS" } - stages { + stages { + stage('Checkout') { agent{ label 'orion-emc'} steps { @@ -20,6 +20,7 @@ pipeline { sh 'git submodule update --init --recursive' } } + stage('Build') { agent{ label 'orion-emc'} steps { @@ -27,9 +28,7 @@ pipeline { sh 'sorc/link_workflow.sh' script { case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - case_list = case_list_output.tokenize('\n') - case_list.remove(0) - cases=case_list + cases = case_list_output.tokenize('\n').remove(0) echo "cases: ${cases}" } } @@ -38,23 +37,25 @@ pipeline { stage('Create Cases') { agent{ label 'orion-emc'} steps { - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') + sh ''' + rm -rf ${RUNTESTS} + mkdir -p ${RUNTESTS} + ''' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + } echo "cases: ${cases}" cases.each { case_name -> stage("Create ${case_name}") { script { env.case = case_name - } + } sh ''' - rm -rf ${RUNTESTS} - mkdir -p ${RUNTESTS} source ci/platforms/config.orion pr_sha=$(git rev-parse --short HEAD) export pslot=${case}_${pr_sha} source workflow/gw_setup.sh - unset HOMEgfs HOMEgfs=${WORKSPACE} workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml ''' @@ -83,4 +84,4 @@ pipeline { } } } -} \ No newline at end of file +} -- GitLab From 6156b342eb11465bc82970ebb4e08e66664c6ce2 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 27 Dec 2023 21:24:11 -0500 Subject: [PATCH 074/447] Update ci_utils_wrapper.sh added HOMEgfs defined inside wrapper --- ci/scripts/utils/ci_utils_wrapper.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh index f3cd461e8..f7cb0fb89 100755 --- a/ci/scripts/utils/ci_utils_wrapper.sh +++ b/ci/scripts/utils/ci_utils_wrapper.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" + utitilty_function="${1}" source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" -${utitilty_function} "${@:2}" \ No newline at end of file +${utitilty_function} "${@:2}" -- GitLab From 9dec9fdcecadd07e4cf619423b8237171fd6856b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 27 Dec 2023 21:27:02 -0500 Subject: [PATCH 075/447] Update Jenkinsfile_org indents --- Jenkinsfile_org | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile_org b/Jenkinsfile_org index 93d168820..fccde67a9 100644 --- a/Jenkinsfile_org +++ b/Jenkinsfile_org @@ -38,8 +38,8 @@ pipeline { agent{ label 'orion-emc'} steps { sh ''' - rm -rf ${RUNTESTS} - mkdir -p ${RUNTESTS} + rm -rf ${RUNTESTS} + mkdir -p ${RUNTESTS} ''' script { pullRequest.removeLabel('CI-Orion-Building') @@ -52,12 +52,12 @@ pipeline { env.case = case_name } sh ''' - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh - HOMEgfs=${WORKSPACE} - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + export pslot=${case}_${pr_sha} + source workflow/gw_setup.sh + HOMEgfs=${WORKSPACE} + workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml ''' script { pullRequest.comment("SUCCESS creating ${case_name} on Orion") -- GitLab From d48e80042c7de1af606ab24043a1a981d085624b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:38:02 +0000 Subject: [PATCH 076/447] Moving Jenkinsfile back in --- Jenkinsfile_org => Jenkinsfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile_org => Jenkinsfile (100%) diff --git a/Jenkinsfile_org b/Jenkinsfile similarity index 100% rename from Jenkinsfile_org rename to Jenkinsfile -- GitLab From 6771e43932b22d2a147eb01d35a201e98a21af37 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:39:33 +0000 Subject: [PATCH 077/447] extra bracket at end removed --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fccde67a9..c09fef80a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,5 +83,4 @@ pipeline { pullRequest.addLabel('CI-Orion-Failed') } } - } -} + } \ No newline at end of file -- GitLab From 6e5d9723c34a6d12fb4df2717b2244fd86ae8214 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:42:32 +0000 Subject: [PATCH 078/447] echo of cases needs to be in script block --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c09fef80a..0ffe87886 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,8 +44,8 @@ pipeline { script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') + echo "cases: ${cases}" } - echo "cases: ${cases}" cases.each { case_name -> stage("Create ${case_name}") { script { -- GitLab From 21af3995651c6988f6fbe3d79f7537395ac44433 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:49:45 +0000 Subject: [PATCH 079/447] put case each back into script block --- Jenkinsfile | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0ffe87886..e87a90bc4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,38 +37,35 @@ pipeline { stage('Create Cases') { agent{ label 'orion-emc'} steps { - sh ''' - rm -rf ${RUNTESTS} - mkdir -p ${RUNTESTS} - ''' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - echo "cases: ${cases}" - } + sh ''' + rm -rf ${RUNTESTS} + mkdir -p ${RUNTESTS} + ''' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + echo "cases: ${cases}" cases.each { case_name -> stage("Create ${case_name}") { script { - env.case = case_name - } - sh ''' - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh + env.case = case_name + } + sh ''' + source ci/platforms/config.orion + pr_sha=$(git rev-parse --short HEAD) + export pslot=${case}_${pr_sha} + source workflow/gw_setup.sh HOMEgfs=${WORKSPACE} workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml - ''' - script { - pullRequest.comment("SUCCESS creating ${case_name} on Orion") - } + ''' + script { + pullRequest.comment("SUCCESS creating ${case_name} on Orion") + } } } } } - } - - } + } post { success { -- GitLab From b62e71c56734d9cf60eaac4927fa034720d4e41d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:51:54 +0000 Subject: [PATCH 080/447] put missing bracket back over stages --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index e87a90bc4..70e02b781 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,6 +66,7 @@ pipeline { } } } + } post { success { -- GitLab From 9041ac42b4324d8fda3f52d46bbd33d1b738fbfa Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:53:48 +0000 Subject: [PATCH 081/447] last bracket over pipeline missing --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70e02b781..3c31ee09c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -81,4 +81,5 @@ pipeline { pullRequest.addLabel('CI-Orion-Failed') } } - } \ No newline at end of file + } +} \ No newline at end of file -- GitLab From 2b056352d1fbb58122ba1c7acce80c1cb819bfc4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 15:56:39 +0000 Subject: [PATCH 082/447] Update stage name and need to comment out build --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c31ee09c..7709e4008 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { stage('Build') { agent{ label 'orion-emc'} steps { - sh 'sorc/build_all.sh' + //sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' script { case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() @@ -34,7 +34,7 @@ pipeline { } } - stage('Create Cases') { + stage('Create Experiments') { agent{ label 'orion-emc'} steps { sh ''' -- GitLab From 28bd6ec88d0cc1423fa379e5386a000624efb3d6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 16:07:54 +0000 Subject: [PATCH 083/447] removed echo of HOMEgfs in get_pr_case_list --- Jenkinsfile | 2 +- ci/scripts/utils/ci_utils.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7709e4008..2e096946e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { sh 'sorc/link_workflow.sh' script { case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list_output.tokenize('\n').remove(0) + cases = case_list_output.tokenize('\n') echo "cases: ${cases}" } } diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index cc9f8aaa2..bec983bc8 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -29,7 +29,6 @@ function get_pr_case_list () { # loop over every yaml file in the PR's ci/cases # and create an run directory for each one for this PR loop ############################################################# - echo ${HOMEgfs} for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true echo "${case}" -- GitLab From a4206f42a0a14a87210b60ea992109ff1f0cabbd Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:18:59 +0000 Subject: [PATCH 084/447] abstracted env setup of experment creatation --- Jenkinsfile | 24 ++++-------------------- ci/scripts/utils/ci_utils_wrapper.sh | 1 + 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e096946e..7740601de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,6 @@ pipeline { script { case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() cases = case_list_output.tokenize('\n') - echo "cases: ${cases}" } } } @@ -37,32 +36,17 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} steps { - sh ''' - rm -rf ${RUNTESTS} - mkdir -p ${RUNTESTS} - ''' + sh 'mkdir -p ${RUNTESTS}' script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') - echo "cases: ${cases}" cases.each { case_name -> stage("Create ${case_name}") { - script { - env.case = case_name - } - sh ''' - source ci/platforms/config.orion - pr_sha=$(git rev-parse --short HEAD) - export pslot=${case}_${pr_sha} - source workflow/gw_setup.sh - HOMEgfs=${WORKSPACE} - workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml - ''' - script { - pullRequest.comment("SUCCESS creating ${case_name} on Orion") - } + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } } } diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh index f7cb0fb89..51c392fb9 100755 --- a/ci/scripts/utils/ci_utils_wrapper.sh +++ b/ci/scripts/utils/ci_utils_wrapper.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +source "${HOMEgfs}/ush/detect_machine.sh" utitilty_function="${1}" -- GitLab From 009a0489df45371535ee7707b1c60abced5c2974 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:21:49 +0000 Subject: [PATCH 085/447] forgot to add updated ci_utils.sh --- ci/scripts/utils/ci_utils.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index bec983bc8..75ea04d08 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -33,4 +33,15 @@ function get_pr_case_list () { case=$(basename "${yaml_config}" .yaml) || true echo "${case}" done +} + +function create_experiment () { + + yaml_config="${1}" + source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" + pr_sha=$(git rev-parse --short HEAD) + export pslot=${case}_${pr_sha} + source "${HOMEgfs}/workflow/gw_setup.sh" + "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + } \ No newline at end of file -- GitLab From ceb9db5c52f2f4c563fcb4a1f5416522e8e203e4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:31:04 +0000 Subject: [PATCH 086/447] triing to add delcrative steps in creating experiments again --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7740601de..5c5f0bdd2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,4 @@ -def cases="cases" pipeline { agent{ label 'orion-emc'} @@ -26,10 +25,6 @@ pipeline { steps { //sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' - script { - case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list_output.tokenize('\n') - } } } @@ -40,10 +35,15 @@ pipeline { script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') cases.each { case_name -> stage("Create ${case_name}") { - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + agent{ label 'orion-emc'} + steps { + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } -- GitLab From 29492b32c8cea6fe44feefbdbc1a63853dd3c906 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:51:01 +0000 Subject: [PATCH 087/447] try stage stage --- Jenkinsfile | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c5f0bdd2..79ec65d94 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,32 +25,29 @@ pipeline { steps { //sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' + sh 'mkdir -p ${RUNTESTS}' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + } } } stage('Create Experiments') { agent{ label 'orion-emc'} - steps { - sh 'mkdir -p ${RUNTESTS}' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - steps { - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } - } - } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + steps { + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } } } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } - } post { success { -- GitLab From 4f48ed4f671f1fcbb75decd2eed3c7d3d7cee826 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:53:47 +0000 Subject: [PATCH 088/447] add parallel --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 79ec65d94..0a5e2534a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,6 +38,7 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} cases.each { case_name -> + parallel { stage("Create ${case_name}") { agent{ label 'orion-emc'} steps { @@ -45,6 +46,7 @@ pipeline { sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } + } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } -- GitLab From 7c788526cd78b3984ec67c1da57c53c43566295e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 17:58:21 +0000 Subject: [PATCH 089/447] missed bracker over all staages --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0a5e2534a..3ebceffa4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,6 +50,7 @@ pipeline { } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } + } post { success { -- GitLab From ce4fa1ea2d1750ac3e378390af64d4657c4ce2f5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:19:56 +0000 Subject: [PATCH 090/447] added parllel over cases --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3ebceffa4..0d81e34c8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,16 +38,16 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} cases.each { case_name -> - parallel { stage("Create ${case_name}") { - agent{ label 'orion-emc'} - steps { - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } + cases[ "${case_name}"] = { + node(label 'orion-emc') { + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } } - } } + parallel cases script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } } -- GitLab From 994f0c642650827d2920acd9da376637d1592ddb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:21:56 +0000 Subject: [PATCH 091/447] replace node with agent --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d81e34c8..8e17a8473 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { cases[ "${case_name}"] = { - node(label 'orion-emc') { + agent(label 'orion-emc') { script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } -- GitLab From d1ae8d3d450825aefa7785ed49f7881324391505 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:24:17 +0000 Subject: [PATCH 092/447] removed agent block --- Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8e17a8473..d405fc2a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,11 +39,10 @@ pipeline { agent{ label 'orion-emc'} cases.each { case_name -> stage("Create ${case_name}") { + agent{ label 'orion-emc'} cases[ "${case_name}"] = { - agent(label 'orion-emc') { - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } } -- GitLab From c1fb33626879da5658134754dd0f39e5800446ea Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:35:04 +0000 Subject: [PATCH 093/447] try global mapp --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d405fc2a9..5ebc68084 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,4 @@ +define cases_map = [:] pipeline { agent{ label 'orion-emc'} @@ -37,16 +38,18 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} + script { cases.each { case_name -> stage("Create ${case_name}") { agent{ label 'orion-emc'} - cases[ "${case_name}"] = { + cases_map[ "${case_name}"] = { script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } } - parallel cases + } + parallel cases_map script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } } -- GitLab From e1902ca757723ab33b555d9cab9b1b7288b11560 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:37:46 +0000 Subject: [PATCH 094/447] added steps --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 5ebc68084..426048b78 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,6 +38,7 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} + steps { script { cases.each { case_name -> stage("Create ${case_name}") { @@ -49,6 +50,7 @@ pipeline { } } } + } parallel cases_map script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } -- GitLab From b0f75b3403b278d3fd0e601c3d4e01b4453802bb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:44:30 +0000 Subject: [PATCH 095/447] revert to what had worked --- Jenkinsfile | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 426048b78..5c5f0bdd2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,3 @@ -define cases_map = [:] pipeline { agent{ label 'orion-emc'} @@ -26,35 +25,32 @@ pipeline { steps { //sh 'sorc/build_all.sh' sh 'sorc/link_workflow.sh' - sh 'mkdir -p ${RUNTESTS}' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - } } } stage('Create Experiments') { agent{ label 'orion-emc'} steps { - script { - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - cases_map[ "${case_name}"] = { - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + sh 'mkdir -p ${RUNTESTS}' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + steps { + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } } - } - } - parallel cases_map - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } - } + } post { success { -- GitLab From 56ee98b84f0f15d9489c4ad4c7f84f249687698a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:47:54 +0000 Subject: [PATCH 096/447] remove steps --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c5f0bdd2..ca9f83620 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,10 +40,8 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { agent{ label 'orion-emc'} - steps { script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } -- GitLab From 730272184f030894bc2b278801732a9af44b4bbb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 18:57:33 +0000 Subject: [PATCH 097/447] try parallel --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index ca9f83620..c53d6fda5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,11 +38,13 @@ pipeline { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() cases = case_list.tokenize('\n') cases.each { case_name -> + parallel { stage("Create ${case_name}") { agent{ label 'orion-emc'} script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } + } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } -- GitLab From e1abb2de8ba14a5b37642c762255643eaf49efb6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:04:57 +0000 Subject: [PATCH 098/447] try parallel run cases again --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c53d6fda5..00ce3e3ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,6 @@ +def run_cases = [:] + pipeline { agent{ label 'orion-emc'} @@ -38,7 +40,7 @@ pipeline { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() cases = case_list.tokenize('\n') cases.each { case_name -> - parallel { + run_cases["${case_name}"] = { stage("Create ${case_name}") { agent{ label 'orion-emc'} script { env.case = case_name } @@ -48,6 +50,7 @@ pipeline { } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } + parallel run_cases } } } -- GitLab From a793bb81b7126612128c35617fd3a9f5dd623aaf Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:08:06 +0000 Subject: [PATCH 099/447] try parallel run cases again still --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 00ce3e3ce..680b18a2a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,8 +40,8 @@ pipeline { case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() cases = case_list.tokenize('\n') cases.each { case_name -> - run_cases["${case_name}"] = { stage("Create ${case_name}") { + run_cases["${case_name}"] = { agent{ label 'orion-emc'} script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' -- GitLab From ef8ee86e33367ca4fb5437dc01ee499de6d17cf0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:10:17 +0000 Subject: [PATCH 100/447] try parallel run cases again still 2 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 680b18a2a..9ec1ddc2f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,8 +49,8 @@ pipeline { } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + parallel run_cases } - parallel run_cases } } } -- GitLab From 0b7313cba905757ad52af88c3a61f7cbdf77ad8d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:18:18 +0000 Subject: [PATCH 101/447] move parallel --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9ec1ddc2f..8f0dd19e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,15 +42,14 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { run_cases["${case_name}"] = { - agent{ label 'orion-emc'} script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } } script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - parallel run_cases } + parallel run_cases } } } -- GitLab From 27bf59190a2318033008d28723f87107c0b4ff0e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:21:15 +0000 Subject: [PATCH 102/447] move parallel yet again --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 8f0dd19e3..ae87b3235 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,8 +42,10 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { run_cases["${case_name}"] = { + node('case-creator') { script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } } } } -- GitLab From 41f00a9fd04d263bc0705b42ea9db989b151070b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:23:26 +0000 Subject: [PATCH 103/447] move parallel yet again --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae87b3235..0e7692097 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,9 +49,9 @@ pipeline { } } } + parallel run_cases script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } - parallel run_cases } } } -- GitLab From 3a4c9bec6371b8c9ad8d8ea4695fd092aaa953e3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:34:41 +0000 Subject: [PATCH 104/447] agent node --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0e7692097..6d37ca7cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,10 +42,10 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { run_cases["${case_name}"] = { - node('case-creator') { + agent { node { label 'case-creator'} { script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } + } } } } -- GitLab From 6b11fc5c3703c0ee4df90391032a9960a06a140a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:37:26 +0000 Subject: [PATCH 105/447] agent node --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6d37ca7cc..2e4443213 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,12 +42,11 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { run_cases["${case_name}"] = { - agent { node { label 'case-creator'} { + agent { node { label 'case-creator'} } script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } - } } parallel run_cases script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } -- GitLab From 1b18b814b7b2b1f013c555e834f12961b1d8af62 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 19:48:24 +0000 Subject: [PATCH 106/447] working version --- Jenkinsfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e4443213..ca9f83620 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,4 @@ -def run_cases = [:] - pipeline { agent{ label 'orion-emc'} @@ -41,14 +39,11 @@ pipeline { cases = case_list.tokenize('\n') cases.each { case_name -> stage("Create ${case_name}") { - run_cases["${case_name}"] = { - agent { node { label 'case-creator'} } + agent{ label 'orion-emc'} script { env.case = case_name } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } } } - parallel run_cases script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } } } -- GitLab From 07ff62c4e0e6182137f9746e9c75b2ba1acb0e29 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 20:04:36 +0000 Subject: [PATCH 107/447] added get pslot list --- ci/scripts/utils/ci_utils.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 75ea04d08..8f75b7ef6 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -35,6 +35,20 @@ function get_pr_case_list () { done } +function get_pslot_list () { + + local RUNTESTS="${1}" + + ############################################################# + # loop over every yaml file in the PR's ci/cases + # and create an run directory for each one for this PR loop + ############################################################# + for pslot in "${RUNTESTS}/EXPDIR/"*; do + echo "${pslot}" + done + +} + function create_experiment () { yaml_config="${1}" -- GitLab From 8c1cd31da07a3b50e1063bf8ac6e9fd30b003a22 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 20:26:33 +0000 Subject: [PATCH 108/447] added (sequential) run experments --- Jenkinsfile | 18 ++++++++++++++++++ ci/scripts/run-check_ci.sh | 2 +- ci/scripts/utils/ci_utils.sh | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca9f83620..9ba74ca78 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,6 +48,24 @@ pipeline { } } } + + stage('Run Experiments') { + agent{ label 'orion-emc'} + steps { + script { + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list", returnStdout: true ).trim() + experiments = experiment_list.tokenize('\n') + experiments.each { experiment_name -> + stage("Run ${experiment_name}") { + agent{ label 'orion-emc'} + script { env.experiment = experiment_name } + sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}' + } + } + script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } + } + } + } } post { diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index a5c5369ef..939ac1b44 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -21,7 +21,7 @@ pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this scr # │  └── ${pslot} # └── EXPDIR # └── ${pslot} -HOMEgfs="${TEST_DIR}/HOMEgfs" +HOMEgfs="${TEST_DIR}" RUNTESTS="${TEST_DIR}/RUNTESTS" # Source modules and setup logging diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 8f75b7ef6..120df5b5c 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -43,7 +43,8 @@ function get_pslot_list () { # loop over every yaml file in the PR's ci/cases # and create an run directory for each one for this PR loop ############################################################# - for pslot in "${RUNTESTS}/EXPDIR/"*; do + for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do + pslot=$(basename "${plsot_dir}") || true echo "${pslot}" done -- GitLab From deb4372567bc475f55dfb209170f21d5247904f9 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 20:28:39 +0000 Subject: [PATCH 109/447] added build all script in build all for real --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9ba74ca78..3742d19da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { stage('Build') { agent{ label 'orion-emc'} steps { - //sh 'sorc/build_all.sh' + sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } -- GitLab From 6409a337719ffa8fe8d0011fd4c32f09273793a2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 28 Dec 2023 22:06:57 +0000 Subject: [PATCH 110/447] forgot path for pslot list --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3742d19da..b51033d22 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { stage('Build') { agent{ label 'orion-emc'} steps { - sh 'sorc/build_all.sh -gu' + //sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } @@ -53,7 +53,7 @@ pipeline { agent{ label 'orion-emc'} steps { script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list", returnStdout: true ).trim() + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim() experiments = experiment_list.tokenize('\n') experiments.each { experiment_name -> stage("Run ${experiment_name}") { -- GitLab From 5c2d107290f24e7f7c052f5b83449870f4fe0925 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 18:17:21 +0000 Subject: [PATCH 111/447] clear logs on controler by renaming Jenkinsfile --- Jenkinsfile => Jenkinsfile_org | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile => Jenkinsfile_org (100%) diff --git a/Jenkinsfile b/Jenkinsfile_org similarity index 100% rename from Jenkinsfile rename to Jenkinsfile_org -- GitLab From 04913f551861eba109b86e72de974b1b2e9d8f48 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 18:35:25 +0000 Subject: [PATCH 112/447] added case in create experiment --- ci/scripts/utils/ci_utils.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 120df5b5c..ab410347a 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -51,12 +51,16 @@ function get_pslot_list () { } function create_experiment () { + + local yaml_config="${1}" + cd "${HOMEgfs}" || exit 1 + pr_sha=$(git rev-parse --short HEAD) + case=$(basename "${yaml_config}" .yaml) || true - yaml_config="${1}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" - pr_sha=$(git rev-parse --short HEAD) - export pslot=${case}_${pr_sha} source "${HOMEgfs}/workflow/gw_setup.sh" + + export pslot=${case}_${pr_sha} "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } \ No newline at end of file -- GitLab From 184045b963be70532c32677f0a06dc8644263ea7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 19:40:26 +0000 Subject: [PATCH 113/447] add Jenkisfile back for new log --- Jenkinsfile_org => Jenkinsfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Jenkinsfile_org => Jenkinsfile (100%) diff --git a/Jenkinsfile_org b/Jenkinsfile similarity index 100% rename from Jenkinsfile_org rename to Jenkinsfile -- GitLab From fd1c7475b1115bc9ebc3c91fb4652d0fa37bc1b7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 19:52:56 +0000 Subject: [PATCH 114/447] typo bug with pslot in get_pslot_list and uncomment build --- Jenkinsfile_org | 85 ++++++++++++++++++++++++++++++++++++ ci/scripts/utils/ci_utils.sh | 2 +- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 Jenkinsfile_org diff --git a/Jenkinsfile_org b/Jenkinsfile_org new file mode 100644 index 000000000..9c00a83f5 --- /dev/null +++ b/Jenkinsfile_org @@ -0,0 +1,85 @@ + +pipeline { + agent{ label 'orion-emc'} + + environment { + RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + + stages { + + stage('Checkout') { + agent{ label 'orion-emc'} + steps { + checkout scm + script { + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') + } + sh 'git submodule update --init --recursive' + } + } + + stage('Build') { + agent{ label 'orion-emc'} + steps { + sh 'sorc/build_all.sh -gu' + sh 'sorc/link_workflow.sh' + } + } + + stage('Create Experiments') { + agent{ label 'orion-emc'} + steps { + sh 'mkdir -p ${RUNTESTS}' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + script { env.case = case_name } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + } + } + } + + stage('Run Experiments') { + agent{ label 'orion-emc'} + steps { + script { + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim() + experiments = experiment_list.tokenize('\n') + experiments.each { experiment_name -> + stage("Run ${experiment_name}") { + agent{ label 'orion-emc'} + script { env.experiment = experiment_name } + sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}' + } + } + script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } + } + } + } + } + + post { + success { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Passed') + } + } + failure { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Failed') + } + } + } +} \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index ab410347a..d46688bd6 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -44,7 +44,7 @@ function get_pslot_list () { # and create an run directory for each one for this PR loop ############################################################# for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do - pslot=$(basename "${plsot_dir}") || true + pslot=$(basename "${pslot_dir}") || true echo "${pslot}" done -- GitLab From 208117d82da6fdfb81450aa19ea1e8108deb4eff Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 20:02:21 +0000 Subject: [PATCH 115/447] bug misspelled ci run script --- Jenkinsfile | 2 +- Jenkinsfile_org | 85 ------------------------------------------------- 2 files changed, 1 insertion(+), 86 deletions(-) delete mode 100644 Jenkinsfile_org diff --git a/Jenkinsfile b/Jenkinsfile index b51033d22..d16695d55 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -59,7 +59,7 @@ pipeline { stage("Run ${experiment_name}") { agent{ label 'orion-emc'} script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}' + sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' } } script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } diff --git a/Jenkinsfile_org b/Jenkinsfile_org deleted file mode 100644 index 9c00a83f5..000000000 --- a/Jenkinsfile_org +++ /dev/null @@ -1,85 +0,0 @@ - -pipeline { - agent{ label 'orion-emc'} - - environment { - RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - - stages { - - stage('Checkout') { - agent{ label 'orion-emc'} - steps { - checkout scm - script { - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') - } - sh 'git submodule update --init --recursive' - } - } - - stage('Build') { - agent{ label 'orion-emc'} - steps { - sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - } - } - - stage('Create Experiments') { - agent{ label 'orion-emc'} - steps { - sh 'mkdir -p ${RUNTESTS}' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - script { env.case = case_name } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } - } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - } - } - } - - stage('Run Experiments') { - agent{ label 'orion-emc'} - steps { - script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim() - experiments = experiment_list.tokenize('\n') - experiments.each { experiment_name -> - stage("Run ${experiment_name}") { - agent{ label 'orion-emc'} - script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}' - } - } - script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } - } - } - } - } - - post { - success { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Passed') - } - } - failure { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Failed') - } - } - } -} \ No newline at end of file -- GitLab From ef880c844e1112e04daefda927e661cabb09c8d2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 20:07:40 +0000 Subject: [PATCH 116/447] added rm RUNTESTS dir before createing experminets at the pipeline level --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index d16695d55..db0609a40 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,6 +31,7 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} steps { + sh 'rm -Rf ${RUNTESTS}' sh 'mkdir -p ${RUNTESTS}' script { pullRequest.removeLabel('CI-Orion-Building') -- GitLab From dcc97f2ae099a779403fdfb218852a32f1debbee Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 21:07:14 +0000 Subject: [PATCH 117/447] removed global RUNTESTS, was getting wrong WORKSPACE dir --- Jenkinsfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index db0609a40..05064b6e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,10 +2,6 @@ pipeline { agent{ label 'orion-emc'} - environment { - RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - stages { stage('Checkout') { @@ -23,7 +19,7 @@ pipeline { stage('Build') { agent{ label 'orion-emc'} steps { - //sh 'sorc/build_all.sh -gu' + sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } @@ -41,7 +37,10 @@ pipeline { cases.each { case_name -> stage("Create ${case_name}") { agent{ label 'orion-emc'} - script { env.case = case_name } + script { + env.case = case_name + env.RUNTESTS = "${WORKSPCE}/RUNTESTS" + } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } @@ -54,7 +53,7 @@ pipeline { agent{ label 'orion-emc'} steps { script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim() + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() experiments = experiment_list.tokenize('\n') experiments.each { experiment_name -> stage("Run ${experiment_name}") { -- GitLab From 3a63a905d2b177d16b427bfc96b9b54937a44163 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 29 Dec 2023 22:18:16 +0000 Subject: [PATCH 118/447] removed RUNTEST var in one more place --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 05064b6e6..a742f3609 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,8 +27,8 @@ pipeline { stage('Create Experiments') { agent{ label 'orion-emc'} steps { - sh 'rm -Rf ${RUNTESTS}' - sh 'mkdir -p ${RUNTESTS}' + sh 'rm -Rf ${WORKSPACE}/RUNTESTS' + sh 'mkdir -p ${WORKSPACE}/RUNTESTS' script { pullRequest.removeLabel('CI-Orion-Building') pullRequest.addLabel('CI-Orion-Running') -- GitLab From b367a1c22073deb91b60c82c4a5f1a040f792dbf Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 30 Dec 2023 00:07:53 +0000 Subject: [PATCH 119/447] SPELLED WORKSPACE WRONG --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a742f3609..142c5bb0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ pipeline { agent{ label 'orion-emc'} script { env.case = case_name - env.RUNTESTS = "${WORKSPCE}/RUNTESTS" + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" } sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } -- GitLab From b9c50edaba1f058c4bd148c5078222d65d39fd18 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 16:57:49 +0000 Subject: [PATCH 120/447] working with matrix construct test case --- Jenkinsfile | 93 +++++++++++------------------------------------- Jenkinsfile_main | 85 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 73 deletions(-) create mode 100644 Jenkinsfile_main diff --git a/Jenkinsfile b/Jenkinsfile index 142c5bb0a..88dba0229 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,85 +1,32 @@ - pipeline { - agent{ label 'orion-emc'} - + agent none stages { - - stage('Checkout') { - agent{ label 'orion-emc'} - steps { - checkout scm - script { - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') + stage('BuildAndTest') { + matrix { + agent any + axes { + axis { + name 'PLATFORM' + values 'Orion', 'Hera', 'Hercules' + } + axis { + name 'Cases' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' + } } - sh 'git submodule update --init --recursive' - } - } - - stage('Build') { - agent{ label 'orion-emc'} - steps { - sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - } - } - - stage('Create Experiments') { - agent{ label 'orion-emc'} - steps { - sh 'rm -Rf ${WORKSPACE}/RUNTESTS' - sh 'mkdir -p ${WORKSPACE}/RUNTESTS' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - script { - env.case = case_name - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + stages { + stage('Build') { + steps { + echo "Do Build for ${PLATFORM}" } } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - } - } - } - - stage('Run Experiments') { - agent{ label 'orion-emc'} - steps { - script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() - experiments = experiment_list.tokenize('\n') - experiments.each { experiment_name -> - stage("Run ${experiment_name}") { - agent{ label 'orion-emc'} - script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' + stage('Run Tests') { + steps { + echo "Do Test for ${PLATFORM} - ${Cases}" } } - script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } } } } - } - - post { - success { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Passed') - } - } - failure { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Failed') - } - } } } \ No newline at end of file diff --git a/Jenkinsfile_main b/Jenkinsfile_main new file mode 100644 index 000000000..142c5bb0a --- /dev/null +++ b/Jenkinsfile_main @@ -0,0 +1,85 @@ + +pipeline { + agent{ label 'orion-emc'} + + stages { + + stage('Checkout') { + agent{ label 'orion-emc'} + steps { + checkout scm + script { + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') + } + sh 'git submodule update --init --recursive' + } + } + + stage('Build') { + agent{ label 'orion-emc'} + steps { + sh 'sorc/build_all.sh -gu' + sh 'sorc/link_workflow.sh' + } + } + + stage('Create Experiments') { + agent{ label 'orion-emc'} + steps { + sh 'rm -Rf ${WORKSPACE}/RUNTESTS' + sh 'mkdir -p ${WORKSPACE}/RUNTESTS' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + script { + env.case = case_name + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + } + } + } + + stage('Run Experiments') { + agent{ label 'orion-emc'} + steps { + script { + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() + experiments = experiment_list.tokenize('\n') + experiments.each { experiment_name -> + stage("Run ${experiment_name}") { + agent{ label 'orion-emc'} + script { env.experiment = experiment_name } + sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' + } + } + script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } + } + } + } + } + + post { + success { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Passed') + } + } + failure { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Failed') + } + } + } +} \ No newline at end of file -- GitLab From d4219d9fd22a3b7528ac60c1180530a70f761d3a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 17:30:27 +0000 Subject: [PATCH 121/447] added correct node name for agent --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 88dba0229..dbdcb3918 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('BuildAndTest') { matrix { - agent any + agent{ label 'orion-emc'} axes { axis { name 'PLATFORM' -- GitLab From 56bd357ea8a55a1274773ad37a532711fe47c676 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 17:33:27 +0000 Subject: [PATCH 122/447] added correct node name for agent in outer loop too --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index dbdcb3918..bedaf953f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent none + agent{ label 'orion-emc'} stages { stage('BuildAndTest') { matrix { -- GitLab From 7e8d366683329e973f4ee1cbc51af2394f306cfe Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 17:36:12 +0000 Subject: [PATCH 123/447] move to working version --- Jenkinsfile | 91 ++++++++++++++++++++++++++++++++++++++---------- Jenkinsfile_main | 85 -------------------------------------------- Jenkinsfile_temp | 32 +++++++++++++++++ 3 files changed, 104 insertions(+), 104 deletions(-) delete mode 100644 Jenkinsfile_main create mode 100644 Jenkinsfile_temp diff --git a/Jenkinsfile b/Jenkinsfile index bedaf953f..142c5bb0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,32 +1,85 @@ + pipeline { agent{ label 'orion-emc'} + stages { - stage('BuildAndTest') { - matrix { - agent{ label 'orion-emc'} - axes { - axis { - name 'PLATFORM' - values 'Orion', 'Hera', 'Hercules' - } - axis { - name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' - } + + stage('Checkout') { + agent{ label 'orion-emc'} + steps { + checkout scm + script { + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') } - stages { - stage('Build') { - steps { - echo "Do Build for ${PLATFORM}" + sh 'git submodule update --init --recursive' + } + } + + stage('Build') { + agent{ label 'orion-emc'} + steps { + sh 'sorc/build_all.sh -gu' + sh 'sorc/link_workflow.sh' + } + } + + stage('Create Experiments') { + agent{ label 'orion-emc'} + steps { + sh 'rm -Rf ${WORKSPACE}/RUNTESTS' + sh 'mkdir -p ${WORKSPACE}/RUNTESTS' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + script { + env.case = case_name + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } - stage('Run Tests') { - steps { - echo "Do Test for ${PLATFORM} - ${Cases}" + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + } + } + } + + stage('Run Experiments') { + agent{ label 'orion-emc'} + steps { + script { + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() + experiments = experiment_list.tokenize('\n') + experiments.each { experiment_name -> + stage("Run ${experiment_name}") { + agent{ label 'orion-emc'} + script { env.experiment = experiment_name } + sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' } } + script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } } } } + } + + post { + success { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Passed') + } + } + failure { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Failed') + } + } } } \ No newline at end of file diff --git a/Jenkinsfile_main b/Jenkinsfile_main deleted file mode 100644 index 142c5bb0a..000000000 --- a/Jenkinsfile_main +++ /dev/null @@ -1,85 +0,0 @@ - -pipeline { - agent{ label 'orion-emc'} - - stages { - - stage('Checkout') { - agent{ label 'orion-emc'} - steps { - checkout scm - script { - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') - } - sh 'git submodule update --init --recursive' - } - } - - stage('Build') { - agent{ label 'orion-emc'} - steps { - sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - } - } - - stage('Create Experiments') { - agent{ label 'orion-emc'} - steps { - sh 'rm -Rf ${WORKSPACE}/RUNTESTS' - sh 'mkdir -p ${WORKSPACE}/RUNTESTS' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - script { - env.case = case_name - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } - } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - } - } - } - - stage('Run Experiments') { - agent{ label 'orion-emc'} - steps { - script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() - experiments = experiment_list.tokenize('\n') - experiments.each { experiment_name -> - stage("Run ${experiment_name}") { - agent{ label 'orion-emc'} - script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' - } - } - script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } - } - } - } - } - - post { - success { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Passed') - } - } - failure { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Failed') - } - } - } -} \ No newline at end of file diff --git a/Jenkinsfile_temp b/Jenkinsfile_temp new file mode 100644 index 000000000..bedaf953f --- /dev/null +++ b/Jenkinsfile_temp @@ -0,0 +1,32 @@ +pipeline { + agent{ label 'orion-emc'} + stages { + stage('BuildAndTest') { + matrix { + agent{ label 'orion-emc'} + axes { + axis { + name 'PLATFORM' + values 'Orion', 'Hera', 'Hercules' + } + axis { + name 'Cases' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' + } + } + stages { + stage('Build') { + steps { + echo "Do Build for ${PLATFORM}" + } + } + stage('Run Tests') { + steps { + echo "Do Test for ${PLATFORM} - ${Cases}" + } + } + } + } + } + } +} \ No newline at end of file -- GitLab From 1d81eba34679906efb81bee6eba2504067396760 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 17:47:39 +0000 Subject: [PATCH 124/447] back to testing matrix locally --- Jenkinsfile | 91 ++++++++++-------------------------------------- Jenkinsfile_main | 85 ++++++++++++++++++++++++++++++++++++++++++++ Jenkinsfile_temp | 32 ----------------- 3 files changed, 104 insertions(+), 104 deletions(-) create mode 100644 Jenkinsfile_main delete mode 100644 Jenkinsfile_temp diff --git a/Jenkinsfile b/Jenkinsfile index 142c5bb0a..bedaf953f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,85 +1,32 @@ - pipeline { agent{ label 'orion-emc'} - stages { - - stage('Checkout') { - agent{ label 'orion-emc'} - steps { - checkout scm - script { - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') + stage('BuildAndTest') { + matrix { + agent{ label 'orion-emc'} + axes { + axis { + name 'PLATFORM' + values 'Orion', 'Hera', 'Hercules' + } + axis { + name 'Cases' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' + } } - sh 'git submodule update --init --recursive' - } - } - - stage('Build') { - agent{ label 'orion-emc'} - steps { - sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - } - } - - stage('Create Experiments') { - agent{ label 'orion-emc'} - steps { - sh 'rm -Rf ${WORKSPACE}/RUNTESTS' - sh 'mkdir -p ${WORKSPACE}/RUNTESTS' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - script { - env.case = case_name - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + stages { + stage('Build') { + steps { + echo "Do Build for ${PLATFORM}" } } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - } - } - } - - stage('Run Experiments') { - agent{ label 'orion-emc'} - steps { - script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() - experiments = experiment_list.tokenize('\n') - experiments.each { experiment_name -> - stage("Run ${experiment_name}") { - agent{ label 'orion-emc'} - script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' + stage('Run Tests') { + steps { + echo "Do Test for ${PLATFORM} - ${Cases}" } } - script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } } } } - } - - post { - success { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Passed') - } - } - failure { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Failed') - } - } } } \ No newline at end of file diff --git a/Jenkinsfile_main b/Jenkinsfile_main new file mode 100644 index 000000000..142c5bb0a --- /dev/null +++ b/Jenkinsfile_main @@ -0,0 +1,85 @@ + +pipeline { + agent{ label 'orion-emc'} + + stages { + + stage('Checkout') { + agent{ label 'orion-emc'} + steps { + checkout scm + script { + pullRequest.removeLabel('CI-Orion-Ready') + pullRequest.addLabel('CI-Orion-Building') + } + sh 'git submodule update --init --recursive' + } + } + + stage('Build') { + agent{ label 'orion-emc'} + steps { + sh 'sorc/build_all.sh -gu' + sh 'sorc/link_workflow.sh' + } + } + + stage('Create Experiments') { + agent{ label 'orion-emc'} + steps { + sh 'rm -Rf ${WORKSPACE}/RUNTESTS' + sh 'mkdir -p ${WORKSPACE}/RUNTESTS' + script { + pullRequest.removeLabel('CI-Orion-Building') + pullRequest.addLabel('CI-Orion-Running') + case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() + cases = case_list.tokenize('\n') + cases.each { case_name -> + stage("Create ${case_name}") { + agent{ label 'orion-emc'} + script { + env.case = case_name + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } + script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } + } + } + } + + stage('Run Experiments') { + agent{ label 'orion-emc'} + steps { + script { + experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() + experiments = experiment_list.tokenize('\n') + experiments.each { experiment_name -> + stage("Run ${experiment_name}") { + agent{ label 'orion-emc'} + script { env.experiment = experiment_name } + sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' + } + } + script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } + } + } + } + } + + post { + success { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Passed') + } + } + failure { + script { + pullRequest.removeLabel('CI-Orion-Running') + pullRequest.addLabel('CI-Orion-Failed') + } + } + } +} \ No newline at end of file diff --git a/Jenkinsfile_temp b/Jenkinsfile_temp deleted file mode 100644 index bedaf953f..000000000 --- a/Jenkinsfile_temp +++ /dev/null @@ -1,32 +0,0 @@ -pipeline { - agent{ label 'orion-emc'} - stages { - stage('BuildAndTest') { - matrix { - agent{ label 'orion-emc'} - axes { - axis { - name 'PLATFORM' - values 'Orion', 'Hera', 'Hercules' - } - axis { - name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' - } - } - stages { - stage('Build') { - steps { - echo "Do Build for ${PLATFORM}" - } - } - stage('Run Tests') { - steps { - echo "Do Test for ${PLATFORM} - ${Cases}" - } - } - } - } - } - } -} \ No newline at end of file -- GitLab From 56e50bd06d6738379e708b848b23108d94c57d71 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 18:07:22 +0000 Subject: [PATCH 125/447] removed inner agent and two platforms for testing --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bedaf953f..ebdd8bc74 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,11 +3,10 @@ pipeline { stages { stage('BuildAndTest') { matrix { - agent{ label 'orion-emc'} axes { axis { name 'PLATFORM' - values 'Orion', 'Hera', 'Hercules' + values 'Orion', 'Hera' } axis { name 'Cases' -- GitLab From 707336e78dce6ca3ea5e4f30bfaa617587e16d6c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 18:10:29 +0000 Subject: [PATCH 126/447] removed inner agent and two platforms for testing still --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ebdd8bc74..c804e2b7b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,12 +14,12 @@ pipeline { } } stages { - stage('Build') { + stage('Build ${PLATFORM}') { steps { echo "Do Build for ${PLATFORM}" } } - stage('Run Tests') { + stage('Run Tests ${PLATFORM} - ${Cases}') { steps { echo "Do Test for ${PLATFORM} - ${Cases}" } -- GitLab From b39af6152a1332def29c2e75ac2c8cd678453261 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 20:17:09 +0000 Subject: [PATCH 127/447] added sample pipeline for arch opt --- Jenkins_arch_opt | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 Jenkins_arch_opt diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt new file mode 100644 index 000000000..3aa31fa53 --- /dev/null +++ b/Jenkins_arch_opt @@ -0,0 +1,38 @@ +pipeline { + agent none + + stages { + parallel { + // The matrix directive would only be available + // in parallel blocks and is used to generate stages in + // a declarative fashion + matrix { + // Options define a map of options we could potentially have, + // each map defines a matrix for each individual stage + opts: [ + [machine: 'emc-orion'], + [machine: 'emc-hera'], + ] + } + + // To access variables defined in our opts we can use the groovy + // standard `it` + stage("${it.machine} Building & Testing") { + agent { label "${it.machine}" } + stage("Build") { + steps { + sh("echo Build on ${it.machine}") + } + } + stage("Run tests") { + steps { + sh("Running Test One on ${it.machine}") + } + steps { + sh("Running Test Two on ${it.machine}") + } + } + } + } + } +} \ No newline at end of file -- GitLab From f74ee472063f697a5d23ea10a00d2aa54d65c2ef Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 20:21:06 +0000 Subject: [PATCH 128/447] missed bracket --- Jenkins_arch_opt | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt index 3aa31fa53..f10858de7 100644 --- a/Jenkins_arch_opt +++ b/Jenkins_arch_opt @@ -28,9 +28,6 @@ pipeline { steps { sh("Running Test One on ${it.machine}") } - steps { - sh("Running Test Two on ${it.machine}") - } } } } -- GitLab From eeb94e723bfed10265fc398259e27e72dba88fcf Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 21:08:48 +0000 Subject: [PATCH 129/447] basic parallel matrix --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c804e2b7b..408d45e01 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,12 +1,13 @@ pipeline { - agent{ label 'orion-emc'} + agent none stages { stage('BuildAndTest') { matrix { + agent { label '${PLATFORM}-emc' } axes { axis { name 'PLATFORM' - values 'Orion', 'Hera' + values 'orion', 'hera' } axis { name 'Cases' @@ -14,12 +15,12 @@ pipeline { } } stages { - stage('Build ${PLATFORM}') { + stage('Build') { steps { - echo "Do Build for ${PLATFORM}" + echo "Do Build for ${env.PLATFORM}" } } - stage('Run Tests ${PLATFORM} - ${Cases}') { + stage('Run Cases') { steps { echo "Do Test for ${PLATFORM} - ${Cases}" } -- GitLab From 95d100e9541a919bbc226dde41e4ab6acc81a5c5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 21:13:26 +0000 Subject: [PATCH 130/447] double quotes in label for platform --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 408d45e01..b7e7e153b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('BuildAndTest') { matrix { - agent { label '${PLATFORM}-emc' } + agent { label "${PLATFORM}-emc" } axes { axis { name 'PLATFORM' @@ -17,7 +17,7 @@ pipeline { stages { stage('Build') { steps { - echo "Do Build for ${env.PLATFORM}" + echo "Do Build for ${PLATFORM}" } } stage('Run Cases') { -- GitLab From b78045cc3c999a92e1489964c2d1a18a3f044120 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 23:42:34 +0000 Subject: [PATCH 131/447] added label based agents in matrix --- Jenkinsfile | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7e7e153b..c67ee68fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,14 +1,38 @@ pipeline { agent none stages { - stage('BuildAndTest') { + + stage( 'Get Machine' ) { + + steps { + script { + for (label in pullRequest.labels) { + if ((label.matches("CI-Hera-Ready"))) { + env.CHOICE_NODE='hera-emc' + } + else if ((label.matches("CI-Orion-Ready"))) { + env.CHOICE_NODE='orion-emc' + } + else if ((label.matches("CI-Hercules-Ready"))) { + env.CHOICE_NODE='hercules-emc' + } + else { + env.CHOICE_NODE='none' + } + } + } + } + } + + stage( 'Build and Test' ) { + + when { + expression { env.CHOICE_NODE != 'none' } + } + matrix { - agent { label "${PLATFORM}-emc" } + agent { label "${CHOICE_NODE}" } axes { - axis { - name 'PLATFORM' - values 'orion', 'hera' - } axis { name 'Cases' values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' -- GitLab From 7946fad108bfe329a7ce0ae34b7c65b4ae937e4f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 3 Jan 2024 23:47:16 +0000 Subject: [PATCH 132/447] removed PLATFORM replaced by CHOICE_NODE --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c67ee68fb..a70096bb7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,12 +41,12 @@ pipeline { stages { stage('Build') { steps { - echo "Do Build for ${PLATFORM}" + echo "Do Build for ${CHOICE_NODE} - ${Cases}" } } stage('Run Cases') { steps { - echo "Do Test for ${PLATFORM} - ${Cases}" + echo "Do Test for ${CHOICE_NODE} - ${Cases}" } } } -- GitLab From 639b4f68d69a1b5fe84d5e1741b464276a00c6a9 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:02:31 +0000 Subject: [PATCH 133/447] added build and create experments steps --- Jenkinsfile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index a70096bb7..5b52e7d98 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,12 @@ pipeline { agent none + + options { + disableConcurrentBuilds() + overrideIndexTriggers(false) + skipDefaultCheckout(true) + } + stages { stage( 'Get Machine' ) { @@ -42,6 +49,23 @@ pipeline { stage('Build') { steps { echo "Do Build for ${CHOICE_NODE} - ${Cases}" + cleaanWs() + checkout scm + sh 'sorc/build_all.sh -gu' + sh 'sorc/link_workflow.sh' + } + } + stage('Create Experiment') { + steps { + script { + env.case = ${Cases} + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + sh ''' + rm -rf ${RUNTESTS} + mkdir -p ${RUNTESTS} + ''' + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } stage('Run Cases') { -- GitLab From 4df67d89802814b84eed06e4d06935cc59e822e5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:04:18 +0000 Subject: [PATCH 134/447] spelled clean wrong --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5b52e7d98..e3377c0be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,7 +49,7 @@ pipeline { stage('Build') { steps { echo "Do Build for ${CHOICE_NODE} - ${Cases}" - cleaanWs() + cleanWs() checkout scm sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' -- GitLab From cdf2cb129f093e1879d6069241da68459ee31901 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:24:43 +0000 Subject: [PATCH 135/447] added MACHINE and updated PR Labels so next platform can run --- Jenkinsfile | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e3377c0be..b5990db10 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,16 +15,16 @@ pipeline { script { for (label in pullRequest.labels) { if ((label.matches("CI-Hera-Ready"))) { - env.CHOICE_NODE='hera-emc' + env.MACHINE='hera' } else if ((label.matches("CI-Orion-Ready"))) { - env.CHOICE_NODE='orion-emc' + env.MACHINE='orion' } else if ((label.matches("CI-Hercules-Ready"))) { - env.CHOICE_NODE='hercules-emc' + env.MACHINE='hercules' } else { - env.CHOICE_NODE='none' + env.MACHINE='none' } } } @@ -34,11 +34,11 @@ pipeline { stage( 'Build and Test' ) { when { - expression { env.CHOICE_NODE != 'none' } + expression { env.MACHINE != 'none' } } matrix { - agent { label "${CHOICE_NODE}" } + agent { label "${MACHINE}-emc" } axes { axis { name 'Cases' @@ -48,10 +48,14 @@ pipeline { stages { stage('Build') { steps { - echo "Do Build for ${CHOICE_NODE} - ${Cases}" + echo "Do Build for ${MACHINE^} - ${Cases}" + script { + pullRequest.removeLabel('CI-${MACHINE^}-Ready') + pullRequest.addLabel('CI-${MACHINE^}-Building') + } cleanWs() checkout scm - sh 'sorc/build_all.sh -gu' + //sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } @@ -68,7 +72,7 @@ pipeline { sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } - stage('Run Cases') { + stage("Run ${Cases}") { steps { echo "Do Test for ${CHOICE_NODE} - ${Cases}" } @@ -77,4 +81,13 @@ pipeline { } } } + + post { + always { + script { + pullRequest.removeLabel('CI-${MACHINE^}-Building') + pullRequest.addLabel('CI-${MACHINE^}-Done') + } + } + } } \ No newline at end of file -- GitLab From 5ed8d5d9a78a270c49b1a3c95803020b50b0bc7d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:44:48 +0000 Subject: [PATCH 136/447] tried to get Uppercase in MACHINE to work --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5990db10..8082b3d5e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,11 +48,12 @@ pipeline { stages { stage('Build') { steps { - echo "Do Build for ${MACHINE^} - ${Cases}" script { - pullRequest.removeLabel('CI-${MACHINE^}-Ready') - pullRequest.addLabel('CI-${MACHINE^}-Building') + machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${machine}-Building") } + echo "Do Build for" ${machine} ${Cases} cleanWs() checkout scm //sh 'sorc/build_all.sh -gu' @@ -85,8 +86,8 @@ pipeline { post { always { script { - pullRequest.removeLabel('CI-${MACHINE^}-Building') - pullRequest.addLabel('CI-${MACHINE^}-Done') + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Done") } } } -- GitLab From 93ae7ba68731ea9260f00d2a21dab4d2a1356934 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:49:03 +0000 Subject: [PATCH 137/447] removed var in stage name --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8082b3d5e..5e3dca40a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,9 +73,9 @@ pipeline { sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } - stage("Run ${Cases}") { + stage("Run Cases") { steps { - echo "Do Test for ${CHOICE_NODE} - ${Cases}" + echo "Do Test for ${machine} - ${Cases}" } } } -- GitLab From 99c23d13367efe0db4337fd68bb3554a8da87a71 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 00:50:44 +0000 Subject: [PATCH 138/447] missed quote in echo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e3dca40a..34c32c555 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") } - echo "Do Build for" ${machine} ${Cases} + echo "Do Build for ${machine} ${Cases}" cleanWs() checkout scm //sh 'sorc/build_all.sh -gu' -- GitLab From 51c1d757ea0f70b42b4e594e07c3d2793ca9fc48 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:03:55 +0000 Subject: [PATCH 139/447] removed wipe --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 34c32c555..af2b0747d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,7 +54,6 @@ pipeline { pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine} ${Cases}" - cleanWs() checkout scm //sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' -- GitLab From 32cd5d7a0fec4852e59fedcac736209790a7c675 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:11:06 +0000 Subject: [PATCH 140/447] echo label --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index af2b0747d..85b9a7f45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,6 +14,7 @@ pipeline { steps { script { for (label in pullRequest.labels) { + echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { env.MACHINE='hera' } -- GitLab From 362b48c97b755190ce8f1ac92f712eca9728dc66 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:13:54 +0000 Subject: [PATCH 141/447] echo labels --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 85b9a7f45..3aa614a7d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,6 +13,7 @@ pipeline { steps { script { + echo pullRequest.labels for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { -- GitLab From b79c13ec9a47a5f471814a3c33e40a0ad1a73f72 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:16:10 +0000 Subject: [PATCH 142/447] echo machine in post --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 3aa614a7d..b8abb3a5a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,6 +87,8 @@ pipeline { post { always { script { + machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + echo "Do Post for ${machine}" pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Done") } -- GitLab From eba62614523a157c62f09760545175de4729b087 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:18:32 +0000 Subject: [PATCH 143/447] echo no op in post --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b8abb3a5a..ea3643a6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,10 +87,11 @@ pipeline { post { always { script { - machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) - echo "Do Post for ${machine}" - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Done") + //machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + //echo "Do Post for ${machine}" + //pullRequest.removeLabel("CI-${machine}-Building") + //pullRequest.addLabel("CI-${machine}-Done") + echo "no-op" } } } -- GitLab From 4d3a87299d2436da8fb90b485bdbfe203687e8bb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:26:51 +0000 Subject: [PATCH 144/447] moved MACHINE out from env --- Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ea3643a6a..c1121db41 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,16 +17,16 @@ pipeline { for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { - env.MACHINE='hera' + MACHINE='hera' } else if ((label.matches("CI-Orion-Ready"))) { - env.MACHINE='orion' + MACHINE='orion' } else if ((label.matches("CI-Hercules-Ready"))) { - env.MACHINE='hercules' + MACHINE='hercules' } else { - env.MACHINE='none' + MACHINE='none' } } } @@ -36,7 +36,7 @@ pipeline { stage( 'Build and Test' ) { when { - expression { env.MACHINE != 'none' } + expression { MACHINE != 'none' } } matrix { @@ -87,10 +87,10 @@ pipeline { post { always { script { - //machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) - //echo "Do Post for ${machine}" - //pullRequest.removeLabel("CI-${machine}-Building") - //pullRequest.addLabel("CI-${machine}-Done") + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + echo "Do Post for ${machine}" + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Passed") echo "no-op" } } -- GitLab From 52e20cd5123c61b3694028a969f5d9567f71bd72 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:29:03 +0000 Subject: [PATCH 145/447] deleted and echo --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c1121db41..342e31e9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,6 @@ pipeline { steps { script { - echo pullRequest.labels for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { -- GitLab From 38713d6a17b5f0615a5b75167020f247c81858d1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:31:26 +0000 Subject: [PATCH 146/447] removed fail label in post --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 342e31e9d..bdeff84b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,9 +88,8 @@ pipeline { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) echo "Do Post for ${machine}" - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Passed") - echo "no-op" + //pullRequest.removeLabel("CI-${machine}-Building") + //pullRequest.addLabel("CI-${machine}-Passed") } } } -- GitLab From cb1240a5b32f80d6faf23de7a0ddfa82de323ed3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:37:33 +0000 Subject: [PATCH 147/447] missed and env on Macine --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bdeff84b8..3905bc8f7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { stage('Build') { steps { script { - machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") } -- GitLab From f8ebd722c5dd202932eb45623e4cc50e11dfbce2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:44:46 +0000 Subject: [PATCH 148/447] do not do else and make MACHINE none --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3905bc8f7..0831a8233 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,6 +13,7 @@ pipeline { steps { script { + MACHINE = 'none' for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { @@ -24,9 +25,6 @@ pipeline { else if ((label.matches("CI-Hercules-Ready"))) { MACHINE='hercules' } - else { - MACHINE='none' - } } } } -- GitLab From 943e5046d437d032cfe973ccf04c8fedb68b57b4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:50:34 +0000 Subject: [PATCH 149/447] add Machine ID --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0831a8233..2c2a42eaf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,6 +55,7 @@ pipeline { echo "Do Build for ${machine} ${Cases}" checkout scm //sh 'sorc/build_all.sh -gu' + script { env.MACHINE_ID = MACHINE } sh 'sorc/link_workflow.sh' } } -- GitLab From 62b10b28e4914fd7c7556b98084d3dcb7db92b0b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 01:55:08 +0000 Subject: [PATCH 150/447] can not remove twice --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2c2a42eaf..469deb74f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,8 +49,8 @@ pipeline { steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel("CI-${machine}-Ready") - pullRequest.addLabel("CI-${machine}-Building") + //pullRequest.removeLabel("CI-${machine}-Ready") + //pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine} ${Cases}" checkout scm -- GitLab From fd9f6b207536e41616099c2f50d7b9027450cb53 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:01:20 +0000 Subject: [PATCH 151/447] needed quotes around create Cases --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 469deb74f..2a1c6fbfc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -62,7 +62,7 @@ pipeline { stage('Create Experiment') { steps { script { - env.case = ${Cases} + env.case = "${Cases}" env.RUNTESTS = "${WORKSPACE}/RUNTESTS" } sh ''' -- GitLab From c952ef0d76e640659df4472b3821c4876cc22600 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:04:40 +0000 Subject: [PATCH 152/447] removed rm RUNTESTS duh --- Jenkinsfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2a1c6fbfc..864bf6153 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,10 +65,7 @@ pipeline { env.case = "${Cases}" env.RUNTESTS = "${WORKSPACE}/RUNTESTS" } - sh ''' - rm -rf ${RUNTESTS} - mkdir -p ${RUNTESTS} - ''' + sh 'mkdir -p ${RUNTESTS}' sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } -- GitLab From 009f89708de3c3a388cabc5fb172e60920acafc4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:11:50 +0000 Subject: [PATCH 153/447] removed hybatmDA --- Jenkinsfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 864bf6153..a2108c2a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ pipeline { axes { axis { name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' //, 'C96C48_hybatmDA' } } stages { @@ -80,12 +80,18 @@ pipeline { } post { - always { + success { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) echo "Do Post for ${machine}" - //pullRequest.removeLabel("CI-${machine}-Building") - //pullRequest.addLabel("CI-${machine}-Passed") + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Passed") + } + failure { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') } } } -- GitLab From 2688bf970b28038c2fbf8d2c8c8935000f40f6eb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:13:21 +0000 Subject: [PATCH 154/447] label change --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a2108c2a6..92553334f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,8 +49,8 @@ pipeline { steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - //pullRequest.removeLabel("CI-${machine}-Ready") - //pullRequest.addLabel("CI-${machine}-Building") + pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine} ${Cases}" checkout scm -- GitLab From 370d328be594879500cf4657bfc0ce01aad08e25 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:18:46 +0000 Subject: [PATCH 155/447] braclets --- Jenkinsfile | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 92553334f..3f746fc93 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ pipeline { axes { axis { name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' //, 'C96C48_hybatmDA' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' } } stages { @@ -78,21 +78,22 @@ pipeline { } } } - - post { - success { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - echo "Do Post for ${machine}" - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Passed") - } - failure { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') - } +} + +post { + success { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + echo "Do Post for ${machine}" + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Passed") + } + } + failure { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') } } } \ No newline at end of file -- GitLab From 46274bc910d5818a3090a69a2e739ee7a252d851 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 4 Jan 2024 02:41:50 +0000 Subject: [PATCH 156/447] added yes to create experemnt --- Jenkinsfile | 2 +- ci/scripts/utils/ci_utils.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3f746fc93..c35947d67 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,7 +54,7 @@ pipeline { } echo "Do Build for ${machine} ${Cases}" checkout scm - //sh 'sorc/build_all.sh -gu' + sh 'sorc/build_all.sh -gu' script { env.MACHINE_ID = MACHINE } sh 'sorc/link_workflow.sh' } diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index d46688bd6..18ed03707 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -61,6 +61,6 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" export pslot=${case}_${pr_sha} - "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + yes | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } \ No newline at end of file -- GitLab From 0583ee12928db1595ca6cc11c27fbd0ef41f1dba Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 18:26:49 +0000 Subject: [PATCH 157/447] moved build step outside of Matrix block so that all cases run against a single build --- Jenkinsfile | 75 ++++++++++++----------------------------------------- 1 file changed, 16 insertions(+), 59 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c35947d67..6123c79a3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,43 +1,32 @@ pipeline { - agent none - - options { - disableConcurrentBuilds() - overrideIndexTriggers(false) - skipDefaultCheckout(true) - } + agent any stages { - - stage( 'Get Machine' ) { - + stage('Build') { + when { + expression { env.MACHINE != 'none' } + } steps { script { - MACHINE = 'none' - for (label in pullRequest.labels) { - echo "Label: ${label}" - if ((label.matches("CI-Hera-Ready"))) { - MACHINE='hera' - } - else if ((label.matches("CI-Orion-Ready"))) { - MACHINE='orion' - } - else if ((label.matches("CI-Hercules-Ready"))) { - MACHINE='hercules' - } - } + machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${machine}-Building") } + echo "Do Build for ${machine}" + checkout scm + sh 'sorc/build_all.sh -gu' + script { env.MACHINE_ID = env.MACHINE } + sh 'sorc/link_workflow.sh' } } - stage( 'Build and Test' ) { - + stage('Build and Test') { when { - expression { MACHINE != 'none' } + expression { env.MACHINE != 'none' } } matrix { - agent { label "${MACHINE}-emc" } + agent { label "${env.MACHINE}-emc" } axes { axis { name 'Cases' @@ -45,20 +34,6 @@ pipeline { } } stages { - stage('Build') { - steps { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel("CI-${machine}-Ready") - pullRequest.addLabel("CI-${machine}-Building") - } - echo "Do Build for ${machine} ${Cases}" - checkout scm - sh 'sorc/build_all.sh -gu' - script { env.MACHINE_ID = MACHINE } - sh 'sorc/link_workflow.sh' - } - } stage('Create Experiment') { steps { script { @@ -78,22 +53,4 @@ pipeline { } } } -} - -post { - success { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - echo "Do Post for ${machine}" - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Passed") - } - } - failure { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') - } - } } \ No newline at end of file -- GitLab From 55f7103e5bb6e6f7f9b748d74d1f4af5b549d79a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 18:44:57 +0000 Subject: [PATCH 158/447] Add machine selection based on labels in Jenkinsfile --- Jenkinsfile | 28 ++++++++++++- Jenkinsfile_arch_opt | 99 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 Jenkinsfile_arch_opt diff --git a/Jenkinsfile b/Jenkinsfile index 6123c79a3..9695e363e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,31 @@ pipeline { - agent any + + options { + disableConcurrentBuilds() + overrideIndexTriggers(false) + skipDefaultCheckout(true) + } + + stage( 'Get Machine' ) { + agent { label 'master' } + steps { + script { + MACHINE = 'none' + for (label in pullRequest.labels) { + echo "Label: ${label}" + if ((label.matches("CI-Hera-Ready"))) { + MACHINE='hera' + } + else if ((label.matches("CI-Orion-Ready"))) { + MACHINE='orion' + } + else if ((label.matches("CI-Hercules-Ready"))) { + MACHINE='hercules' + } + } + } + } + } stages { stage('Build') { diff --git a/Jenkinsfile_arch_opt b/Jenkinsfile_arch_opt new file mode 100644 index 000000000..c35947d67 --- /dev/null +++ b/Jenkinsfile_arch_opt @@ -0,0 +1,99 @@ +pipeline { + agent none + + options { + disableConcurrentBuilds() + overrideIndexTriggers(false) + skipDefaultCheckout(true) + } + + stages { + + stage( 'Get Machine' ) { + + steps { + script { + MACHINE = 'none' + for (label in pullRequest.labels) { + echo "Label: ${label}" + if ((label.matches("CI-Hera-Ready"))) { + MACHINE='hera' + } + else if ((label.matches("CI-Orion-Ready"))) { + MACHINE='orion' + } + else if ((label.matches("CI-Hercules-Ready"))) { + MACHINE='hercules' + } + } + } + } + } + + stage( 'Build and Test' ) { + + when { + expression { MACHINE != 'none' } + } + + matrix { + agent { label "${MACHINE}-emc" } + axes { + axis { + name 'Cases' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' + } + } + stages { + stage('Build') { + steps { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${machine}-Building") + } + echo "Do Build for ${machine} ${Cases}" + checkout scm + sh 'sorc/build_all.sh -gu' + script { env.MACHINE_ID = MACHINE } + sh 'sorc/link_workflow.sh' + } + } + stage('Create Experiment') { + steps { + script { + env.case = "${Cases}" + env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + } + sh 'mkdir -p ${RUNTESTS}' + sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + } + } + stage("Run Cases") { + steps { + echo "Do Test for ${machine} - ${Cases}" + } + } + } + } + } + } +} + +post { + success { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + echo "Do Post for ${machine}" + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Passed") + } + } + failure { + script { + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') + } + } +} \ No newline at end of file -- GitLab From 502ee8e603fb1e6677d094be48993341d5ba30a5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 18:48:37 +0000 Subject: [PATCH 159/447] Add agent label for build stage and global master --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 9695e363e..ce6ce7f05 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,6 @@ pipeline { + agent { label 'master' } options { disableConcurrentBuilds() overrideIndexTriggers(false) @@ -29,6 +30,7 @@ pipeline { stages { stage('Build') { + agent { label "${env.MACHINE}-emc" } when { expression { env.MACHINE != 'none' } } -- GitLab From 404259f39c45acb813aee0d7ccc5bbb812411483 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 18:57:17 +0000 Subject: [PATCH 160/447] moved main stages block to top --- Jenkinsfile | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ce6ce7f05..16462cbea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,28 +7,27 @@ pipeline { skipDefaultCheckout(true) } - stage( 'Get Machine' ) { - agent { label 'master' } - steps { - script { - MACHINE = 'none' - for (label in pullRequest.labels) { - echo "Label: ${label}" - if ((label.matches("CI-Hera-Ready"))) { - MACHINE='hera' - } - else if ((label.matches("CI-Orion-Ready"))) { - MACHINE='orion' - } - else if ((label.matches("CI-Hercules-Ready"))) { - MACHINE='hercules' - } + stages { + + stage('Get Machine') { + agent { label 'master' } + steps { + script { + MACHINE = 'none' + for (label in pullRequest.labels) { + echo "Label: ${label}" + if ((label.matches("CI-Hera-Ready"))) { + MACHINE = 'hera' + } else if ((label.matches("CI-Orion-Ready"))) { + MACHINE = 'orion' + } else if ((label.matches("CI-Hercules-Ready"))) { + MACHINE = 'hercules' + } + } } } } - } - stages { stage('Build') { agent { label "${env.MACHINE}-emc" } when { @@ -43,7 +42,7 @@ pipeline { echo "Do Build for ${machine}" checkout scm sh 'sorc/build_all.sh -gu' - script { env.MACHINE_ID = env.MACHINE } + script { env.MACHINE_ID = env.MACHINE } sh 'sorc/link_workflow.sh' } } @@ -58,13 +57,13 @@ pipeline { axes { axis { name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' + values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' } } stages { stage('Create Experiment') { steps { - script { + script { env.case = "${Cases}" env.RUNTESTS = "${WORKSPACE}/RUNTESTS" } @@ -80,5 +79,6 @@ pipeline { } } } + } } \ No newline at end of file -- GitLab From a40a55a5f21580c8d82d862b1927daa91613b3c4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 19:11:43 +0000 Subject: [PATCH 161/447] none agent on top and buit-in before assinged remote agent --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 16462cbea..2e4824c6c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { - agent { label 'master' } + agent none options { disableConcurrentBuilds() overrideIndexTriggers(false) @@ -10,7 +10,7 @@ pipeline { stages { stage('Get Machine') { - agent { label 'master' } + agent { label 'build-in' } steps { script { MACHINE = 'none' -- GitLab From c0b8c4fcade67c33a9398ac9c11ad5a0b212f79b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 19:14:25 +0000 Subject: [PATCH 162/447] build in to buit in type fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e4824c6c..878de6789 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { stages { stage('Get Machine') { - agent { label 'build-in' } + agent { label 'build-it' } steps { script { MACHINE = 'none' -- GitLab From 34be579bcda0c50d092bdc7a4a09cbf353b9c0ce Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 19:16:30 +0000 Subject: [PATCH 163/447] still typed buid-in wrong git add Jenkinsfile ! grrr --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 878de6789..2e4824c6c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { stages { stage('Get Machine') { - agent { label 'build-it' } + agent { label 'build-in' } steps { script { MACHINE = 'none' -- GitLab From 3ef2866ad59b9b8ec0f4d5993c68f912b9b713ad Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 19:18:09 +0000 Subject: [PATCH 164/447] OMG I typed built-in yet one more time --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e4824c6c..1f1ef218f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { stages { stage('Get Machine') { - agent { label 'build-in' } + agent { label 'built-in' } steps { script { MACHINE = 'none' -- GitLab From e22f33404c6c85c0dc75c6b0668b78104685513a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 19:24:36 +0000 Subject: [PATCH 165/447] moved MACHINE out of env scpope and make global to Jenkins --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f1ef218f..082a41f42 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,27 +29,27 @@ pipeline { } stage('Build') { - agent { label "${env.MACHINE}-emc" } + agent { label "${MACHINE}-emc" } when { - expression { env.MACHINE != 'none' } + expression { MACHINE != 'none' } } steps { script { - machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1) + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine}" checkout scm + script { env.MACHINE_ID = MACHINE } sh 'sorc/build_all.sh -gu' - script { env.MACHINE_ID = env.MACHINE } sh 'sorc/link_workflow.sh' } } stage('Build and Test') { when { - expression { env.MACHINE != 'none' } + expression { MACHINE != 'none' } } matrix { -- GitLab From 632f8f7be943a720a9081df77349745ac11f08d5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 20:24:36 +0000 Subject: [PATCH 166/447] left env on last global MACHINE var in creating the experments --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 082a41f42..2d0774743 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { } matrix { - agent { label "${env.MACHINE}-emc" } + agent { label "${MACHINE}-emc" } axes { axis { name 'Cases' -- GitLab From 89e4dc756407a27f098f03c8efd5ecb93385d751 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 20:25:58 +0000 Subject: [PATCH 167/447] skip actual buiding for testing --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2d0774743..fe4c31ff7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { echo "Do Build for ${machine}" checkout scm script { env.MACHINE_ID = MACHINE } - sh 'sorc/build_all.sh -gu' + //sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } -- GitLab From b64f45759af7b0fff4e7a5e1fb8b80490ac14edd Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 20:45:10 +0000 Subject: [PATCH 168/447] skipDefaultCheckout may had wrong syntax, also removed yes pip to create experement --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe4c31ff7..a63c3e4c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,9 +2,9 @@ pipeline { agent none options { - disableConcurrentBuilds() - overrideIndexTriggers(false) - skipDefaultCheckout(true) + disableConcurrentBuilds(abortProcess: true) + skipDefaultCheckout() + buildDicarder(logRotator(numToKeepStr: '2')) } stages { @@ -42,7 +42,7 @@ pipeline { echo "Do Build for ${machine}" checkout scm script { env.MACHINE_ID = MACHINE } - //sh 'sorc/build_all.sh -gu' + // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' } } -- GitLab From 82b69ff48fa7e69641cbc5e082a87a6bada2de70 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 20:47:34 +0000 Subject: [PATCH 169/447] updated to abortPrevious --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a63c3e4c1..f29634365 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent none options { - disableConcurrentBuilds(abortProcess: true) + disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout() buildDicarder(logRotator(numToKeepStr: '2')) } -- GitLab From a7826e6c90fa91c15392a79ee3b4e011af326196 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 20:50:03 +0000 Subject: [PATCH 170/447] spelled buildDiscarder wrong and fixed --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f29634365..6a5df28e2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ pipeline { options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout() - buildDicarder(logRotator(numToKeepStr: '2')) + buildDiscarder(logRotator(numToKeepStr: '2')) } stages { -- GitLab From 599ad702f6e20c39ecf9168dd4e2b1c627a63af3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 21:05:34 +0000 Subject: [PATCH 171/447] remove the workspace path and leave it to the common sc area --- Jenkinsfile | 4 ++-- ci/scripts/utils/ci_utils.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6a5df28e2..e2ca57dd3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,10 +65,10 @@ pipeline { steps { script { env.case = "${Cases}" - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" + env.RUNTESTS = "RUNTESTS" } sh 'mkdir -p ${RUNTESTS}' - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + sh 'ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' } } stage("Run Cases") { diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 18ed03707..d46688bd6 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -61,6 +61,6 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" export pslot=${case}_${pr_sha} - yes | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } \ No newline at end of file -- GitLab From c6a9f84d99d271dd1267b9546638f92fbc75a58a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 21:18:37 +0000 Subject: [PATCH 172/447] trying to force HOMEgfs from scm WORKSPACE --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e2ca57dd3..c99307937 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,6 +44,7 @@ pipeline { script { env.MACHINE_ID = MACHINE } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' + HOMEgfs="${WORKSPACE}" } } @@ -66,9 +67,10 @@ pipeline { script { env.case = "${Cases}" env.RUNTESTS = "RUNTESTS" + env.HOMEgfs = "${HOMEgfs}" } - sh 'mkdir -p ${RUNTESTS}' - sh 'ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' + sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}' + sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml' } } stage("Run Cases") { -- GitLab From 264363ad2025f3148e1eee9623c9345fcc84478c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 21:22:40 +0000 Subject: [PATCH 173/447] moved assignment of Jenkins HOMEgfs into sript block --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c99307937..35cb41a94 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,10 +41,12 @@ pipeline { } echo "Do Build for ${machine}" checkout scm - script { env.MACHINE_ID = MACHINE } + script { + env.MACHINE_ID = MACHINE + HOMEgfs="${WORKSPACE}" + } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' - HOMEgfs="${WORKSPACE}" } } -- GitLab From d07c1abd4f9de6258d3b5f743a43e95fce7d449b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 21:40:12 +0000 Subject: [PATCH 174/447] moved common mkdir for RUNCASES dir and hope matrix is resetting env.case --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35cb41a94..28d35af1e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,11 +42,13 @@ pipeline { echo "Do Build for ${machine}" checkout scm script { + HOMEgfs = "${WORKSPACE}" + env.HOMEgfs = "${HOMEgfs}" env.MACHINE_ID = MACHINE - HOMEgfs="${WORKSPACE}" } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' + sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}' } } @@ -71,7 +73,6 @@ pipeline { env.RUNTESTS = "RUNTESTS" env.HOMEgfs = "${HOMEgfs}" } - sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}' sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml' } } -- GitLab From 58192282b288a05ddfa4b14f5c2b8d11ee9778ac Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 21:57:08 +0000 Subject: [PATCH 175/447] degugging Case control variable in shell --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28d35af1e..7cc418a29 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' - sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}' + sh 'mkdir -p ${HOMEgfs}/RUNTESTS' } } @@ -61,8 +61,8 @@ pipeline { agent { label "${MACHINE}-emc" } axes { axis { - name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' + name "Cases" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } stages { @@ -70,9 +70,10 @@ pipeline { steps { script { env.case = "${Cases}" - env.RUNTESTS = "RUNTESTS" env.HOMEgfs = "${HOMEgfs}" } + echo "Cases: ${Cases}" + sh 'echo "Create Experiment for ${case} and ${Cases}"' sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml' } } -- GitLab From 74eae6d461b82edf970b08a10c0906ba41e20d15 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:00:17 +0000 Subject: [PATCH 176/447] Cases does work in shell --- Jenkinsfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7cc418a29..d732b821e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,13 +68,8 @@ pipeline { stages { stage('Create Experiment') { steps { - script { - env.case = "${Cases}" - env.HOMEgfs = "${HOMEgfs}" - } - echo "Cases: ${Cases}" - sh 'echo "Create Experiment for ${case} and ${Cases}"' - sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml' + sh 'echo "Create Experiment for ${Cases}"' + sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml' } } stage("Run Cases") { -- GitLab From 678e01817e6b2a11774d28be02f0d44538c8edb0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:11:26 +0000 Subject: [PATCH 177/447] issue with HOMEgfs in env --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d732b821e..8133cf294 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,7 +68,8 @@ pipeline { stages { stage('Create Experiment') { steps { - sh 'echo "Create Experiment for ${Cases}"' + script { env.HOMEgfs = "${HOMEgfs}" } + echo "Cases: ${Cases}" sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml' } } -- GitLab From a9de618b0857713c5fca684a8e5bd0c3839aa121 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:15:27 +0000 Subject: [PATCH 178/447] move HOMEgfs to HOME --- Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8133cf294..d0f7f1530 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,8 +42,7 @@ pipeline { echo "Do Build for ${machine}" checkout scm script { - HOMEgfs = "${WORKSPACE}" - env.HOMEgfs = "${HOMEgfs}" + HOME = "${WORKSPACE}" env.MACHINE_ID = MACHINE } // sh 'sorc/build_all.sh -gu' @@ -68,9 +67,8 @@ pipeline { stages { stage('Create Experiment') { steps { - script { env.HOMEgfs = "${HOMEgfs}" } echo "Cases: ${Cases}" - sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml' + sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } } stage("Run Cases") { -- GitLab From d5304ed283b31b91b64819506cc0678085a5ae0a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:17:40 +0000 Subject: [PATCH 179/447] left an extra HOMEgfs in place for mkdir --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d0f7f1530..7a6293009 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,7 +47,7 @@ pipeline { } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' - sh 'mkdir -p ${HOMEgfs}/RUNTESTS' + sh 'mkdir -p ${HOME}/RUNTESTS' } } -- GitLab From 66e507265e21aa40b5be3be799e25bb67a1e0755 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:21:27 +0000 Subject: [PATCH 180/447] seemed to loose HOME --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7a6293009..3824c3471 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -67,7 +67,8 @@ pipeline { stages { stage('Create Experiment') { steps { - echo "Cases: ${Cases}" + script { env.HOME = "$HOME"} + echo "Cases: ${Cases} ${HOME}" sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } } -- GitLab From bbef554e4e34a029f34cf4cf11210d2fe6782adc Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:42:27 +0000 Subject: [PATCH 181/447] need RUNTESTS in shell env --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3824c3471..1368d8248 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,11 +47,11 @@ pipeline { } // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' - sh 'mkdir -p ${HOME}/RUNTESTS' + sh 'mkdir -p ${WORKSPACE}/RUNTESTS' } } - stage('Build and Test') { + stage('Run Tests') { when { expression { MACHINE != 'none' } } @@ -67,12 +67,14 @@ pipeline { stages { stage('Create Experiment') { steps { - script { env.HOME = "$HOME"} + script { + env.HOME = "$HOME" + env.RUNTESTS = "${HOME}/RUNTESTS" echo "Cases: ${Cases} ${HOME}" sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } } - stage("Run Cases") { + stage("Run Experiment") { steps { echo "Do Test for ${machine} - ${Cases}" } -- GitLab From fbcec52c4d82b9296fb91f6a2385193013997600 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 9 Jan 2024 22:44:43 +0000 Subject: [PATCH 182/447] missed a bracket on setting local env --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 1368d8248..1097d3686 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,6 +70,7 @@ pipeline { script { env.HOME = "$HOME" env.RUNTESTS = "${HOME}/RUNTESTS" + } echo "Cases: ${Cases} ${HOME}" sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } -- GitLab From 01267ed873616f508d4d991f746617836ecf631c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 20:15:22 +0000 Subject: [PATCH 183/447] uncommented out the actual build --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1097d3686..5d8503943 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { HOME = "${WORKSPACE}" env.MACHINE_ID = MACHINE } - // sh 'sorc/build_all.sh -gu' + sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' sh 'mkdir -p ${WORKSPACE}/RUNTESTS' } -- GitLab From e2fe2b283fc82d002d55b764ea90c54107743242 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 22:27:14 +0000 Subject: [PATCH 184/447] added Jeinkins code for running experiments within the matrix block --- Jenkinsfile | 12 ++++++++---- ci/scripts/utils/ci_utils.sh | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5d8503943..c9fb15c60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,14 +75,18 @@ pipeline { sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } } - stage("Run Experiment") { - steps { - echo "Do Test for ${machine} - ${Cases}" + + stage('Run Experiments') { + script { + pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' + pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") } } } } - } + } + } } \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index d46688bd6..06090f410 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -40,8 +40,8 @@ function get_pslot_list () { local RUNTESTS="${1}" ############################################################# - # loop over every yaml file in the PR's ci/cases - # and create an run directory for each one for this PR loop + # loop over expdir directories in RUNTESTS + # and create list of the directory names (pslot) with the hash tag ############################################################# for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do pslot=$(basename "${pslot_dir}") || true @@ -50,6 +50,26 @@ function get_pslot_list () { } +function get_pslot () { + + local RUNTESTS="${1}" + local case="${2}" + + ############################################################# + # loop over expdir directories in RUNTESTS + # and return the name of the pslot with its tag that matches the case + ############################################################# + for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do + pslot=$(basename "${pslot_dir}") + check_case = $(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true + if [[ "${check_case}" == "${case}" ]]; then + echo "${pslot}" + break + fi + done + +} + function create_experiment () { local yaml_config="${1}" -- GitLab From 26d8c32a721b1500a8b9779e8768ef3042cec487 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 22:34:39 +0000 Subject: [PATCH 185/447] type in bash assign statment for check_case --- ci/scripts/utils/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 06090f410..5716654c1 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -61,7 +61,7 @@ function get_pslot () { ############################################################# for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do pslot=$(basename "${pslot_dir}") - check_case = $(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true + check_case=$(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true if [[ "${check_case}" == "${case}" ]]; then echo "${pslot}" break -- GitLab From 90be03e214372e4b151cb91fcd8d81f358d3ace5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 22:46:00 +0000 Subject: [PATCH 186/447] added lables for Running and Pass/Fail --- Jenkinsfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c9fb15c60..f7188e577 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -79,14 +79,30 @@ pipeline { stage('Run Experiments') { script { pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.removeLabel('CI-${machine}-Building') + pullRequest.addLabel('CI-${machine}-Running') sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") } } } } - } + } + post { + success { + script { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Passed') + } + } + failure { + script { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') + } + } + } } } \ No newline at end of file -- GitLab From cd11a82d53776428c50cc8ecadf3adbe7bc9ac09 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 22:49:55 +0000 Subject: [PATCH 187/447] missed a bracket placment for stages block to seperate from post --- Jenkinsfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f7188e577..60d075f74 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,21 +88,21 @@ pipeline { } } } + } - post { - success { - script { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Passed') - } + post { + success { + script { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Passed') } - failure { - script { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') - } + } + failure { + script { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') } - } - + } } -} \ No newline at end of file + +} -- GitLab From 2ef90391e329b66eb326a86646da058fc48ccba1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 10 Jan 2024 22:54:53 +0000 Subject: [PATCH 188/447] needed steps in Run stage --- Jenkinsfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 60d075f74..2e44bd305 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,12 +77,14 @@ pipeline { } stage('Run Experiments') { - script { - pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.removeLabel('CI-${machine}-Building') - pullRequest.addLabel('CI-${machine}-Running') + steps { + script { + pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.removeLabel('CI-${machine}-Building') + pullRequest.addLabel('CI-${machine}-Running') + } sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' - pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") + script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") } } } } -- GitLab From 71e3c0060628d57beecaf53a85a60ee4f0b3537c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 00:12:15 +0000 Subject: [PATCH 189/447] removed labels to debug --- Jenkinsfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e44bd305..7fd07e261 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,8 +80,8 @@ pipeline { steps { script { pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.removeLabel('CI-${machine}-Building') - pullRequest.addLabel('CI-${machine}-Running') + //pullRequest.removeLabel('CI-${machine}-Building') + //pullRequest.addLabel('CI-${machine}-Running') } sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") } @@ -95,14 +95,16 @@ pipeline { post { success { script { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Passed') + echo "SUCCESS" + //pullRequest.removeLabel('CI-${machine}-Running') + //pullRequest.addLabel('CI-${machine}-Passed') } } failure { script { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') + echo "FAILURE" + //pullRequest.removeLabel('CI-${machine}-Running') + //pullRequest.addLabel('CI-${machine}-Failed') } } } -- GitLab From 2cf68f577cdd9c02d97dae2fea92690c3dd1fa22 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 00:59:27 +0000 Subject: [PATCH 190/447] Need to rethink Labeling because of the parallel nature of the matrix block --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7fd07e261..7f6dcd208 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,8 +36,8 @@ pipeline { steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel("CI-${machine}-Ready") - pullRequest.addLabel("CI-${machine}-Building") + //pullRequest.removeLabel("CI-${machine}-Ready") + //pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine}" checkout scm @@ -83,8 +83,9 @@ pipeline { //pullRequest.removeLabel('CI-${machine}-Building') //pullRequest.addLabel('CI-${machine}-Running') } + script { pullRequest.comment("Running experiments: ${Case} on ${machine}") } sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' - script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") } + script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } } } -- GitLab From c3997537f1236049efa3569b6ba915a5d14c1f4b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 00:59:53 +0000 Subject: [PATCH 191/447] Need to rethink Labeling because of the parallel nature of the matrix block --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7f6dcd208..0aea22e5a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { HOME = "${WORKSPACE}" env.MACHINE_ID = MACHINE } - sh 'sorc/build_all.sh -gu' + //sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' sh 'mkdir -p ${WORKSPACE}/RUNTESTS' } -- GitLab From fcf0e2816d061506dd4b94bb270fb16ec02cfaab Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 01:08:14 +0000 Subject: [PATCH 192/447] Trying to figure out Run Experiments block is complaining about not having steps, moved agent --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0aea22e5a..5481a951d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,12 +52,12 @@ pipeline { } stage('Run Tests') { + agent { label "${MACHINE}-emc" } when { expression { MACHINE != 'none' } } - matrix { - agent { label "${MACHINE}-emc" } + //agent { label "${MACHINE}-emc" } axes { axis { name "Cases" @@ -82,8 +82,8 @@ pipeline { pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() //pullRequest.removeLabel('CI-${machine}-Building') //pullRequest.addLabel('CI-${machine}-Running') + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") } - script { pullRequest.comment("Running experiments: ${Case} on ${machine}") } sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From c7b69daf53203849381d3522e3963442e160f118 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 01:11:29 +0000 Subject: [PATCH 193/447] moved agent back and coslidated script lines still get weird message about no steps in stage when they are clearly there --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5481a951d..9aa494c35 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,12 +52,11 @@ pipeline { } stage('Run Tests') { - agent { label "${MACHINE}-emc" } when { expression { MACHINE != 'none' } } matrix { - //agent { label "${MACHINE}-emc" } + agent { label "${MACHINE}-emc" } axes { axis { name "Cases" @@ -75,7 +74,6 @@ pipeline { sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' } } - stage('Run Experiments') { steps { script { -- GitLab From 2ce0078a56abd3d3fe8578d0408ab089e0e0a737 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 01:26:46 +0000 Subject: [PATCH 194/447] changed Cases to Case and fixed one that was a mismatch --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9aa494c35..d1911b06e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -59,7 +59,7 @@ pipeline { agent { label "${MACHINE}-emc" } axes { axis { - name "Cases" + name "Case" values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } @@ -70,14 +70,16 @@ pipeline { env.HOME = "$HOME" env.RUNTESTS = "${HOME}/RUNTESTS" } - echo "Cases: ${Cases} ${HOME}" - sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml' + echo "Case: ${Case} ${HOME}" + sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml' } } stage('Run Experiments') { steps { script { + env.HOME = "$HOME" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + env.pslot = "$pslot" //pullRequest.removeLabel('CI-${machine}-Building') //pullRequest.addLabel('CI-${machine}-Running') pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") -- GitLab From aa20993a4b956a9b6e323518d9f16d20d1b7e26a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 01:29:42 +0000 Subject: [PATCH 195/447] replaced WORKSPACE holdover to HOME --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d1911b06e..379eafcf0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ pipeline { //pullRequest.addLabel('CI-${machine}-Running') pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") } - sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' + sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } } -- GitLab From 29ee450803ed4db5989b9fe7215231ee64dd05ba Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 01:46:44 +0000 Subject: [PATCH 196/447] clean workspace --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 379eafcf0..eaba64f00 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,12 +40,13 @@ pipeline { //pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine}" + cleanWs() checkout scm script { HOME = "${WORKSPACE}" env.MACHINE_ID = MACHINE } - //sh 'sorc/build_all.sh -gu' + sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' sh 'mkdir -p ${WORKSPACE}/RUNTESTS' } -- GitLab From 943a33e195a2ba98adc839e7b66f65b55a3f59db Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 17:21:51 +0000 Subject: [PATCH 197/447] Added PR Label setting for Running in non matrixed build block and added checks for Running on post --- Jenkinsfile | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eaba64f00..f0099fb9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,8 +36,8 @@ pipeline { steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - //pullRequest.removeLabel("CI-${machine}-Ready") - //pullRequest.addLabel("CI-${machine}-Building") + pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine}" cleanWs() @@ -49,6 +49,10 @@ pipeline { sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' sh 'mkdir -p ${WORKSPACE}/RUNTESTS' + script { + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Running") + } } } @@ -81,8 +85,6 @@ pipeline { env.HOME = "$HOME" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() env.pslot = "$pslot" - //pullRequest.removeLabel('CI-${machine}-Building') - //pullRequest.addLabel('CI-${machine}-Running') pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") } sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' @@ -97,16 +99,18 @@ pipeline { post { success { script { - echo "SUCCESS" - //pullRequest.removeLabel('CI-${machine}-Running') - //pullRequest.addLabel('CI-${machine}-Passed') + if(pullRequest.labels.contains("CI-${machine}-Running")) { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Passed') + } } } failure { script { - echo "FAILURE" - //pullRequest.removeLabel('CI-${machine}-Running') - //pullRequest.addLabel('CI-${machine}-Failed') + if(pullRequest.labels.contains("CI-${machine}-Running")) { + pullRequest.removeLabel('CI-${machine}-Running') + pullRequest.addLabel('CI-${machine}-Failed') + } } } } -- GitLab From 95c24316ea706ba404c5555a0a833fd8a618f307 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 17:25:37 +0000 Subject: [PATCH 198/447] remove old referencd Jenkinsfiles --- Jenkins_arch_opt | 35 -------------------- Jenkinsfile_main | 85 ------------------------------------------------ 2 files changed, 120 deletions(-) delete mode 100644 Jenkins_arch_opt delete mode 100644 Jenkinsfile_main diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt deleted file mode 100644 index f10858de7..000000000 --- a/Jenkins_arch_opt +++ /dev/null @@ -1,35 +0,0 @@ -pipeline { - agent none - - stages { - parallel { - // The matrix directive would only be available - // in parallel blocks and is used to generate stages in - // a declarative fashion - matrix { - // Options define a map of options we could potentially have, - // each map defines a matrix for each individual stage - opts: [ - [machine: 'emc-orion'], - [machine: 'emc-hera'], - ] - } - - // To access variables defined in our opts we can use the groovy - // standard `it` - stage("${it.machine} Building & Testing") { - agent { label "${it.machine}" } - stage("Build") { - steps { - sh("echo Build on ${it.machine}") - } - } - stage("Run tests") { - steps { - sh("Running Test One on ${it.machine}") - } - } - } - } - } -} \ No newline at end of file diff --git a/Jenkinsfile_main b/Jenkinsfile_main deleted file mode 100644 index 142c5bb0a..000000000 --- a/Jenkinsfile_main +++ /dev/null @@ -1,85 +0,0 @@ - -pipeline { - agent{ label 'orion-emc'} - - stages { - - stage('Checkout') { - agent{ label 'orion-emc'} - steps { - checkout scm - script { - pullRequest.removeLabel('CI-Orion-Ready') - pullRequest.addLabel('CI-Orion-Building') - } - sh 'git submodule update --init --recursive' - } - } - - stage('Build') { - agent{ label 'orion-emc'} - steps { - sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - } - } - - stage('Create Experiments') { - agent{ label 'orion-emc'} - steps { - sh 'rm -Rf ${WORKSPACE}/RUNTESTS' - sh 'mkdir -p ${WORKSPACE}/RUNTESTS' - script { - pullRequest.removeLabel('CI-Orion-Building') - pullRequest.addLabel('CI-Orion-Running') - case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim() - cases = case_list.tokenize('\n') - cases.each { case_name -> - stage("Create ${case_name}") { - agent{ label 'orion-emc'} - script { - env.case = case_name - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } - } - script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") } - } - } - } - - stage('Run Experiments') { - agent{ label 'orion-emc'} - steps { - script { - experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim() - experiments = experiment_list.tokenize('\n') - experiments.each { experiment_name -> - stage("Run ${experiment_name}") { - agent{ label 'orion-emc'} - script { env.experiment = experiment_name } - sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}' - } - } - script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") } - } - } - } - } - - post { - success { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Passed') - } - } - failure { - script { - pullRequest.removeLabel('CI-Orion-Running') - pullRequest.addLabel('CI-Orion-Failed') - } - } - } -} \ No newline at end of file -- GitLab From 75d1db4bff7b4a1ccb658f0c1180305f9f27d953 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 11 Jan 2024 17:28:07 +0000 Subject: [PATCH 199/447] removed another Jenkins reference file --- Jenkinsfile_arch_opt | 99 -------------------------------------------- 1 file changed, 99 deletions(-) delete mode 100644 Jenkinsfile_arch_opt diff --git a/Jenkinsfile_arch_opt b/Jenkinsfile_arch_opt deleted file mode 100644 index c35947d67..000000000 --- a/Jenkinsfile_arch_opt +++ /dev/null @@ -1,99 +0,0 @@ -pipeline { - agent none - - options { - disableConcurrentBuilds() - overrideIndexTriggers(false) - skipDefaultCheckout(true) - } - - stages { - - stage( 'Get Machine' ) { - - steps { - script { - MACHINE = 'none' - for (label in pullRequest.labels) { - echo "Label: ${label}" - if ((label.matches("CI-Hera-Ready"))) { - MACHINE='hera' - } - else if ((label.matches("CI-Orion-Ready"))) { - MACHINE='orion' - } - else if ((label.matches("CI-Hercules-Ready"))) { - MACHINE='hercules' - } - } - } - } - } - - stage( 'Build and Test' ) { - - when { - expression { MACHINE != 'none' } - } - - matrix { - agent { label "${MACHINE}-emc" } - axes { - axis { - name 'Cases' - values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' - } - } - stages { - stage('Build') { - steps { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel("CI-${machine}-Ready") - pullRequest.addLabel("CI-${machine}-Building") - } - echo "Do Build for ${machine} ${Cases}" - checkout scm - sh 'sorc/build_all.sh -gu' - script { env.MACHINE_ID = MACHINE } - sh 'sorc/link_workflow.sh' - } - } - stage('Create Experiment') { - steps { - script { - env.case = "${Cases}" - env.RUNTESTS = "${WORKSPACE}/RUNTESTS" - } - sh 'mkdir -p ${RUNTESTS}' - sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml' - } - } - stage("Run Cases") { - steps { - echo "Do Test for ${machine} - ${Cases}" - } - } - } - } - } - } -} - -post { - success { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - echo "Do Post for ${machine}" - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Passed") - } - } - failure { - script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') - } - } -} \ No newline at end of file -- GitLab From 8a34f421b5034642061dfda3560539b38d4de2a7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 12 Jan 2024 17:15:04 +0000 Subject: [PATCH 200/447] updated cancel_slurm to a generlized cancel_batch for pbs on wcoss --- ci/scripts/utils/ci_utils.sh | 46 ++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 5716654c1..e125fb54b 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -1,28 +1,44 @@ #!/bin/env bash -function cancel_slurm_jobs() { - - # Usage: cancel_slurm_jobs <substring> - # Example: cancel_slurm_jobs "C48_ATM_3c4e7f74" +function cancel_batch_jobs() { + # Usage: cancel_batch_jobs <substring> + # Example: cancel_batch_jobs "C48_ATM_3c4e7f74" # - # Cancel all Slurm jobs that have the given substring in their name + # Cancel all batch jobs that have the given substring in their name # So like in the example all jobs with "C48_ATM_3c4e7f74" # in their name will be canceled local substring=$1 local job_ids - job_ids=$(squeue -u "${USER}" -h -o "%i") - - for job_id in ${job_ids}; do - job_name=$(sacct -j "${job_id}" --format=JobName%100 | head -3 | tail -1 | sed -r 's/\s+//g') || true - if [[ "${job_name}" =~ ${substring} ]]; then - echo "Canceling Slurm Job ${job_name} with: scancel ${job_id}" - scancel "${job_id}" - continue - fi - done + + # cancel pbs jobs <substring> + if [[ ${MACHINE_ID} == "wcoss2" ]]; then + job_ids=$(qstat -u "${USER}" | awk '{print $1}') + + for job_id in ${job_ids}; do + job_name=$(qstat -f "${job_id}" | grep Job_Name | awk '{print $3}') || true + if [[ "${job_name}" =~ ${substring} ]]; then + echo "Canceling PBS Job ${job_name} with: qdel ${job_id}" + qdel "${job_id}" + continue + fi + done + # cancel slurm jobs <substring> + else + job_ids=$(squeue -u "${USER}" -h -o "%i") + + for job_id in ${job_ids}; do + job_name=$(sacct -j "${job_id}" --format=JobName%100 | head -3 | tail -1 | sed -r 's/\s+//g') || true + if [[ "${job_name}" =~ ${substring} ]]; then + echo "Canceling Slurm Job ${job_name} with: scancel ${job_id}" + scancel "${job_id}" + continue + fi + done + fi } + function get_pr_case_list () { ############################################################# -- GitLab From 1d74f242f591226de8e98bc86ab97dec0e3f8f39 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 20:32:51 +0000 Subject: [PATCH 201/447] added some post clean up and messaging --- Jenkinsfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f0099fb9d..d24088835 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { agent none options { disableConcurrentBuilds(abortPrevious: true) - skipDefaultCheckout() + skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) } @@ -103,7 +103,10 @@ pipeline { pullRequest.removeLabel('CI-${machine}-Running') pullRequest.addLabel('CI-${machine}-Passed') } + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") } + cleanWs() } failure { script { @@ -111,6 +114,8 @@ pipeline { pullRequest.removeLabel('CI-${machine}-Running') pullRequest.addLabel('CI-${machine}-Failed') } + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") } } } -- GitLab From 143c196db5e2904d37be19560661fa6f3ff55514 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 22:16:38 +0000 Subject: [PATCH 202/447] needed Case to be in shell env as well as in Jenkins script environment (turned off clean and build in build block --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d24088835..ae83fd13c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { - agent none + agent { label 'built-in' } options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) @@ -40,13 +40,13 @@ pipeline { pullRequest.addLabel("CI-${machine}-Building") } echo "Do Build for ${machine}" - cleanWs() + // cleanWs() checkout scm script { HOME = "${WORKSPACE}" env.MACHINE_ID = MACHINE } - sh 'sorc/build_all.sh -gu' + // sh 'sorc/build_all.sh -gu' sh 'sorc/link_workflow.sh' sh 'mkdir -p ${WORKSPACE}/RUNTESTS' script { @@ -83,6 +83,7 @@ pipeline { steps { script { env.HOME = "$HOME" + env.Case = "$Case" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() env.pslot = "$pslot" pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") -- GitLab From d7d6a79bb8b1958a7ed729d82d3bf1bfec42124f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 22:33:02 +0000 Subject: [PATCH 203/447] still not getting the correct pslot in the matrix, remove shell reference and assigned shell env to variable directly --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae83fd13c..04728b519 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,7 @@ pipeline { stage('Create Experiment') { steps { script { - env.HOME = "$HOME" + env.HOME = HOME env.RUNTESTS = "${HOME}/RUNTESTS" } echo "Case: ${Case} ${HOME}" @@ -82,10 +82,10 @@ pipeline { stage('Run Experiments') { steps { script { - env.HOME = "$HOME" - env.Case = "$Case" + env.HOME = "${HOME}" + env.Case = "${Case}" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - env.pslot = "$pslot" + env.pslot = pslot pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") } sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' -- GitLab From fcf18d13125ab2f6e5a633eba048d081395fad14 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 22:48:29 +0000 Subject: [PATCH 204/447] still not getting the correct pslot in the matrix, moved shell sh into Jenkins script block --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 04728b519..56ed743c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,9 +87,10 @@ pipeline { pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() env.pslot = pslot pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } - sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' - script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } + // sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' } } } -- GitLab From cc3905210cd1d9ac6cfde4b8c1f485a16b64a2ca Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 22:52:48 +0000 Subject: [PATCH 205/447] finally getting pslot in the shell, checking if env statements are needed --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 56ed743c2..1f96737a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,10 +82,10 @@ pipeline { stage('Run Experiments') { steps { script { - env.HOME = "${HOME}" - env.Case = "${Case}" + //env.HOME = "${HOME}" + //env.Case = "${Case}" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - env.pslot = pslot + //env.pslot = pslot pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") -- GitLab From fc4dcb2fe0aee4767608f23d7ec0e490948a5a0e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 23:00:15 +0000 Subject: [PATCH 206/447] remove env assignment statements since all shell commands to the workspace is now done with Jenkins script commands --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f96737a9..dc89b5f17 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,15 +82,11 @@ pipeline { stage('Run Experiments') { steps { script { - //env.HOME = "${HOME}" - //env.Case = "${Case}" pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - //env.pslot = pslot pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } - // sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}' } } } -- GitLab From 0d535d214e97c6fd0b40fb75dbcb1a430ce1064b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 23:29:35 +0000 Subject: [PATCH 207/447] consolodated all the bash shells into Jenkins scripts with the sh method to force it to be declartive --- Jenkinsfile | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc89b5f17..06131389f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,18 +38,13 @@ pipeline { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") - } - echo "Do Build for ${machine}" - // cleanWs() - checkout scm - script { - HOME = "${WORKSPACE}" + cleanWs() + checkout scm + HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE - } - // sh 'sorc/build_all.sh -gu' - sh 'sorc/link_workflow.sh' - sh 'mkdir -p ${WORKSPACE}/RUNTESTS' - script { + sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/link_workflow.sh", returnStatus: false) + sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } @@ -69,29 +64,31 @@ pipeline { } } stages { + ws("${HOMEgfs}") { stage('Create Experiment') { steps { script { - env.HOME = HOME - env.RUNTESTS = "${HOME}/RUNTESTS" + env.HOME = HOMEgfs + env.RUNTESTS = "${HOMEgfs}/RUNTESTS" + sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) } - echo "Case: ${Case} ${HOME}" - sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml' } } stage('Run Experiments') { steps { script { - pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } } + } } } - } + } + } post { -- GitLab From 07ffbf42111da092a8509531dced2759c4f96570 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 16 Jan 2024 23:48:29 +0000 Subject: [PATCH 208/447] moved ws under steps --- Jenkinsfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 06131389f..8dd9ecf46 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -64,27 +64,28 @@ pipeline { } } stages { - ws("${HOMEgfs}") { stage('Create Experiment') { steps { - script { - env.HOME = HOMEgfs - env.RUNTESTS = "${HOMEgfs}/RUNTESTS" - sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) + ws("${HOMEgfs}") { + script { + env.RUNTESTS = "${HOMEgfs}/RUNTESTS" + sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) + } } } } stage('Run Experiments') { steps { - script { - pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + ws("${HOMEgfs}") { + script { + pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + } } } } - } } } } -- GitLab From d70b7acf249e743693f4106904034be07162599c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 00:50:40 +0000 Subject: [PATCH 209/447] moved HOME and ws around --- Jenkinsfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8dd9ecf46..616f7f91b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { checkout scm HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE - sh( script: "sorc/build_all.sh -gu", returnStatus: false) + //sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") @@ -66,21 +66,23 @@ pipeline { stages { stage('Create Experiment') { steps { - ws("${HOMEgfs}") { + ws(HOMEgfs) { script { - env.RUNTESTS = "${HOMEgfs}/RUNTESTS" - sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) + env.HOME=HOMEgfs + env.RUNTESTS = "${HOME}/RUNTESTS" + sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) } } } } stage('Run Experiments') { steps { - ws("${HOMEgfs}") { + ws(HOMEgfs) { script { - pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() + env.HOME=HOMEgfs + pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + sh( script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From e15b033276286669b80428b4d92822061fe89d42 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:01:48 +0000 Subject: [PATCH 210/447] added HOME for common build into full paths of scripts --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 616f7f91b..2c4ca55c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,7 @@ pipeline { script { env.HOME=HOMEgfs env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) + sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) } } } @@ -80,9 +80,9 @@ pipeline { ws(HOMEgfs) { script { env.HOME=HOMEgfs - pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From 2d0c5008515e65cb686813196470db9dc85ab33e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:12:28 +0000 Subject: [PATCH 211/447] added global envirment trying to track down where decltartive fails --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2c4ca55c3..dbce61554 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,6 +7,11 @@ pipeline { buildDiscarder(logRotator(numToKeepStr: '2')) } + environment { + HOMEgfs = "${WORKSPACE}" + MACHINE = 'none' + } + stages { stage('Get Machine') { @@ -68,7 +73,6 @@ pipeline { steps { ws(HOMEgfs) { script { - env.HOME=HOMEgfs env.RUNTESTS = "${HOME}/RUNTESTS" sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) } @@ -79,7 +83,6 @@ pipeline { steps { ws(HOMEgfs) { script { - env.HOME=HOMEgfs pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) -- GitLab From 7e17063a8867ab91835564fae90038bea22e4510 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:24:56 +0000 Subject: [PATCH 212/447] removed glogal environment and when blocks --- Jenkinsfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dbce61554..7efac8247 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,11 +7,6 @@ pipeline { buildDiscarder(logRotator(numToKeepStr: '2')) } - environment { - HOMEgfs = "${WORKSPACE}" - MACHINE = 'none' - } - stages { stage('Get Machine') { @@ -35,9 +30,9 @@ pipeline { stage('Build') { agent { label "${MACHINE}-emc" } - when { - expression { MACHINE != 'none' } - } + //when { + // expression { MACHINE != 'none' } + //} steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) @@ -57,9 +52,9 @@ pipeline { } stage('Run Tests') { - when { - expression { MACHINE != 'none' } - } + //when { + // expression { MACHINE != 'none' } + //} matrix { agent { label "${MACHINE}-emc" } axes { -- GitLab From f30c678e89f052ae25b97d312d4ce384e4d38258 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:32:13 +0000 Subject: [PATCH 213/447] added HOMEgfs in shell (not sh) because it was picking up /home/tmcguinness from sh --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7efac8247..bd4e401e9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,8 +68,8 @@ pipeline { steps { ws(HOMEgfs) { script { - env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false) + env.RUNTESTS = "${gfsHOME}/RUNTESTS" + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) } } } @@ -78,9 +78,9 @@ pipeline { steps { ws(HOMEgfs) { script { - pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From b62fac741d19876ce615a8080442e2a4e0c3474c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:40:51 +0000 Subject: [PATCH 214/447] oops had a misspelling of gfsHOME --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd4e401e9..70fcb5368 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,7 +68,7 @@ pipeline { steps { ws(HOMEgfs) { script { - env.RUNTESTS = "${gfsHOME}/RUNTESTS" + env.RUNTESTS = "${HOMEgfs}/RUNTESTS" sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) } } -- GitLab From 129c13537175f3138eb7dc03678189ab11c959ab Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 01:53:37 +0000 Subject: [PATCH 215/447] put the actual build back in --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70fcb5368..f26734011 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,9 +30,9 @@ pipeline { stage('Build') { agent { label "${MACHINE}-emc" } - //when { - // expression { MACHINE != 'none' } - //} + when { + expression { MACHINE != 'none' } + } steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) @@ -42,7 +42,7 @@ pipeline { checkout scm HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE - //sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") @@ -52,9 +52,9 @@ pipeline { } stage('Run Tests') { - //when { - // expression { MACHINE != 'none' } - //} + when { + expression { MACHINE != 'none' } + } matrix { agent { label "${MACHINE}-emc" } axes { -- GitLab From 77d20dd758bff9e41198aba55808023093a12e51 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 17 Jan 2024 10:48:57 -0500 Subject: [PATCH 216/447] commented out sm and build --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f26734011..459267dfe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,11 +38,11 @@ pipeline { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") - cleanWs() - checkout scm + //cleanWs() + //checkout scm HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE - sh( script: "sorc/build_all.sh -gu", returnStatus: false) + //sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") -- GitLab From c9c37229581c1a856e1b06c25f04a68807039bd6 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 17 Jan 2024 10:57:31 -0500 Subject: [PATCH 217/447] put back in scm and checkout to prime script locations on restart --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 459267dfe..c217dc27c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,8 +38,8 @@ pipeline { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") - //cleanWs() - //checkout scm + cleanWs() + checkout scm HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE //sh( script: "sorc/build_all.sh -gu", returnStatus: false) -- GitLab From 64a37db13edf45455d46b5a234453126ea400578 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 17 Jan 2024 16:51:02 +0000 Subject: [PATCH 218/447] made two shared values between stages global in attempts to make pipeline declartive --- Jenkinsfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c217dc27c..daba1188a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,6 @@ +def MACHINE = 'none' +def HOMEgfs = 'initial value' + pipeline { agent { label 'built-in' } @@ -42,7 +45,7 @@ pipeline { checkout scm HOMEgfs = "${WORKSPACE}" env.MACHINE_ID = MACHINE - //sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") @@ -60,7 +63,8 @@ pipeline { axes { axis { name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_ATM", "C48_S2SW" } } stages { -- GitLab From a7ac197f24c330bb4a81b65beae4a80e74af0e4d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Thu, 18 Jan 2024 20:38:56 +0000 Subject: [PATCH 219/447] Update ci_utils.sh or true for qstat for job_ids shell check --- ci/scripts/utils/ci_utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index e125fb54b..a0b45d5d5 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -13,7 +13,7 @@ function cancel_batch_jobs() { # cancel pbs jobs <substring> if [[ ${MACHINE_ID} == "wcoss2" ]]; then - job_ids=$(qstat -u "${USER}" | awk '{print $1}') + job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true for job_id in ${job_ids}; do job_name=$(qstat -f "${job_id}" | grep Job_Name | awk '{print $3}') || true @@ -99,4 +99,4 @@ function create_experiment () { export pslot=${case}_${pr_sha} "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" -} \ No newline at end of file +} -- GitLab From 154dd055f1c1ad9e502004aef34c0d2487619966 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 21:52:25 +0000 Subject: [PATCH 220/447] working on state added BUILD semaphore file in sorc dir --- Jenkinsfile | 35 ++++++++++++++++++++++----------- ci/scripts/run-check_ci_stub.sh | 5 +++++ sorc/build_all_stub.sh | 5 +++++ 3 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 ci/scripts/run-check_ci_stub.sh create mode 100644 sorc/build_all_stub.sh diff --git a/Jenkinsfile b/Jenkinsfile index daba1188a..88f5bba45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,9 +33,9 @@ pipeline { stage('Build') { agent { label "${MACHINE}-emc" } - when { - expression { MACHINE != 'none' } - } + //when { + // expression { MACHINE != 'none' } + //} steps { script { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) @@ -44,8 +44,18 @@ pipeline { cleanWs() checkout scm HOMEgfs = "${WORKSPACE}" + } + script { env.MACHINE_ID = MACHINE - sh( script: "sorc/build_all.sh -gu", returnStatus: false) + if (not fileExists("${HOMEgfs}/sorc/BUILT_sema")) { + //sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/build_all_stub.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) + } + else { + HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") @@ -55,9 +65,9 @@ pipeline { } stage('Run Tests') { - when { - expression { MACHINE != 'none' } - } + //when { + // expression { MACHINE != 'none' } + //} matrix { agent { label "${MACHINE}-emc" } axes { @@ -84,7 +94,8 @@ pipeline { script { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } @@ -100,9 +111,9 @@ pipeline { success { script { if(pullRequest.labels.contains("CI-${machine}-Running")) { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Passed') + pullRequest.removeLabel("CI-${machine}-Running") } + pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") } @@ -111,9 +122,9 @@ pipeline { failure { script { if(pullRequest.labels.contains("CI-${machine}-Running")) { - pullRequest.removeLabel('CI-${machine}-Running') - pullRequest.addLabel('CI-${machine}-Failed') + pullRequest.removeLabel("CI-${machine}-Running") } + pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") } diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh new file mode 100644 index 000000000..f2903b9a2 --- /dev/null +++ b/ci/scripts/run-check_ci_stub.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eu + +echo -e "HOMEgfs: $1\npslpot: $2\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log \ No newline at end of file diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh new file mode 100644 index 000000000..9acf802c4 --- /dev/null +++ b/sorc/build_all_stub.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eu + +echo "Ran build all stub in ${PWD}" > BUILD_STUB_RAN_$(date +%d%H%M) \ No newline at end of file -- GitLab From 82627415736d8d3ad493338eb398c2a103bed9f3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 21:58:58 +0000 Subject: [PATCH 221/447] reversed logic for BUILT file semaphore --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 88f5bba45..22495ac7b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,15 +47,15 @@ pipeline { } script { env.MACHINE_ID = MACHINE - if (not fileExists("${HOMEgfs}/sorc/BUILT_sema")) { + if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) { + HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } + else { //sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/build_all_stub.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) } - else { - HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - } sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") -- GitLab From 5a0c52abbb9560e5cdc492c9c856d922e9ae06fe Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 22:05:47 +0000 Subject: [PATCH 222/447] added chmod x to sub scripts --- ci/scripts/run-check_ci_stub.sh | 0 sorc/build_all_stub.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/scripts/run-check_ci_stub.sh mode change 100644 => 100755 sorc/build_all_stub.sh diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh old mode 100644 new mode 100755 diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh old mode 100644 new mode 100755 -- GitLab From bf8896bf43da1b02fa57ef8764867ff47208beb4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 22:36:56 +0000 Subject: [PATCH 223/447] needed to set HOMEgfs in run stub --- ci/scripts/run-check_ci_stub.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index f2903b9a2..191bce5d9 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -2,4 +2,7 @@ set -eu -echo -e "HOMEgfs: $1\npslpot: $2\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log \ No newline at end of file +HOMEgfs=$1 +pslpot=$2 + +echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log \ No newline at end of file -- GitLab From 7e60d2357f0d58e4144907e65f182752a42780c8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 22:54:50 +0000 Subject: [PATCH 224/447] needed to set HOMEgfs in run stub and looking at pullRequest.labels.contains --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 22495ac7b..0d67414af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,6 +50,9 @@ pipeline { if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) { HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + //sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/build_all_stub.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) } else { //sh( script: "sorc/build_all.sh -gu", returnStatus: false) @@ -110,7 +113,7 @@ pipeline { post { success { script { - if(pullRequest.labels.contains("CI-${machine}-Running")) { + if ( pullRequest.labels.contains( 'CI-${machine}-Running') ) { pullRequest.removeLabel("CI-${machine}-Running") } pullRequest.addLabel("CI-${machine}-Passed") -- GitLab From 12e3126e5908898c5c0027ac193c89a980cc947e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 23:01:14 +0000 Subject: [PATCH 225/447] typoed pslot in run check stub script --- ci/scripts/run-check_ci_stub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index 191bce5d9..01975f566 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -3,6 +3,6 @@ set -eu HOMEgfs=$1 -pslpot=$2 +pslot=$2 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log \ No newline at end of file -- GitLab From 8f8e8922fa15e415ef6b1bb0cdfd43c5a64a0dda Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 23:40:19 +0000 Subject: [PATCH 226/447] hard coded node name --- Jenkinsfile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d67414af..9b2c5e4b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { } stage('Build') { - agent { label "${MACHINE}-emc" } + agent { label "Orion-emc" } //when { // expression { MACHINE != 'none' } //} @@ -41,7 +41,6 @@ pipeline { machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") - cleanWs() checkout scm HOMEgfs = "${WORKSPACE}" } @@ -50,9 +49,6 @@ pipeline { if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) { HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - //sh( script: "sorc/build_all.sh -gu", returnStatus: false) - sh( script: "sorc/build_all_stub.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) } else { //sh( script: "sorc/build_all.sh -gu", returnStatus: false) @@ -72,7 +68,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "${MACHINE}-emc" } + agent { label "Orion-emc" } axes { axis { name "Case" @@ -113,9 +109,10 @@ pipeline { post { success { script { - if ( pullRequest.labels.contains( 'CI-${machine}-Running') ) { + if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) { pullRequest.removeLabel("CI-${machine}-Running") } + pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") @@ -126,7 +123,8 @@ pipeline { script { if(pullRequest.labels.contains("CI-${machine}-Running")) { pullRequest.removeLabel("CI-${machine}-Running") - } + } + pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") -- GitLab From 94f841767d58fc5cfde30004bbe18e802f2d6da0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 18 Jan 2024 23:53:26 +0000 Subject: [PATCH 227/447] hard coded node names typo capped --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9b2c5e4b6..d9ad451bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { } stage('Build') { - agent { label "Orion-emc" } + agent { label "orion-emc" } //when { // expression { MACHINE != 'none' } //} @@ -68,7 +68,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "Orion-emc" } + agent { label "orion-emc" } axes { axis { name "Case" -- GitLab From 0489383ac058840edb1fecd693221e95e657e240 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 00:13:47 +0000 Subject: [PATCH 228/447] returnStdout on cat BUIT semaphore didnt save --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9ad451bd..d67432013 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,12 +47,12 @@ pipeline { script { env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) { - HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim() + HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { //sh( script: "sorc/build_all.sh -gu", returnStatus: false) - sh( script: "sorc/build_all_stub.sh", returnStatus: false) + sh( script: "sorc/build_all_stub.sh" ) sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) } sh( script: "sorc/link_workflow.sh", returnStatus: false) -- GitLab From 68d245b02c0b7bbe6f4a6ae02bfa5b7c844a1e94 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 16:55:47 +0000 Subject: [PATCH 229/447] added node properties --- Jenkinsfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index d67432013..7b580b1fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,12 @@ def HOMEgfs = 'initial value' pipeline { + properties([parameters([[$class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + defaultSlaves: ['built-in'], name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases']])]) + agent { label 'built-in' } options { disableConcurrentBuilds(abortPrevious: true) -- GitLab From 0834c3f93c803e4b8b37ed7f919c809efef47c18 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:14:50 +0000 Subject: [PATCH 230/447] restored configurable node --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7b580b1fd..9d7cbf59d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ pipeline { } stage('Build') { - agent { label "orion-emc" } + agent { label "${machine}-emc" } //when { // expression { MACHINE != 'none' } //} @@ -74,7 +74,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "orion-emc" } + agent { label "${machine}-emc" } axes { axis { name "Case" -- GitLab From d76e1f1d8cff334de3d0ffbc5fdd007e3e62ddfb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:26:19 +0000 Subject: [PATCH 231/447] moved parameters into options block --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9d7cbf59d..b42920cb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,14 +3,13 @@ def HOMEgfs = 'initial value' pipeline { - properties([parameters([[$class: 'NodeParameterDefinition', - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - defaultSlaves: ['built-in'], name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases']])]) - agent { label 'built-in' } options { + parameters([[$class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + defaultSlaves: ['built-in'], name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases']]) disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) -- GitLab From 5471c46eaef99e12ac6958077af89db5fdd30776 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:36:40 +0000 Subject: [PATCH 232/447] added just parameters section --- Jenkinsfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b42920cb9..db32cef5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,14 +2,15 @@ def MACHINE = 'none' def HOMEgfs = 'initial value' pipeline { - agent { label 'built-in' } + + parameters{[[$class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases']]} + options { - parameters([[$class: 'NodeParameterDefinition', - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - defaultSlaves: ['built-in'], name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases']]) disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) -- GitLab From 8d7d4b2c173e0bb6d9e3ed78915396a0963d9548 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:40:34 +0000 Subject: [PATCH 233/447] moved parameters into options --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index db32cef5d..20f484559 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,13 +4,12 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{[[$class: 'NodeParameterDefinition', - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases']]} - options { + parameters(class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + defaultSlaves: ['built-in'], name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases') disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) -- GitLab From abb747009a7f9da567e0683566445d60794f75f3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:43:55 +0000 Subject: [PATCH 234/447] parameters can not be in options --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 20f484559..f1a544461 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,15 +5,16 @@ pipeline { agent { label 'built-in' } options { - parameters(class: 'NodeParameterDefinition', - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - defaultSlaves: ['built-in'], name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases') disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) } + parameters{class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + defaultSlaves: ['built-in'], name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases' + } stages { -- GitLab From c46fdd68a7d3f79b0d57bbea8e0d6d7765108f13 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:48:37 +0000 Subject: [PATCH 235/447] removed class in parameters --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f1a544461..4ac5aba76 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ pipeline { skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) } - parameters{class: 'NodeParameterDefinition', + parameters{ allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], -- GitLab From 52bd9736a94a13c64d925817b7664bcfe6dd9e59 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:51:45 +0000 Subject: [PATCH 236/447] added class in bracketrs --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ac5aba76..ce16ef46e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,17 +4,16 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } + parameters{[[$class: 'NodeParameterDefinition', + allowedSlaves: ['Hera-EMC', 'Orion-EMC'], + defaultSlaves: ['built-in'], name: 'rdhpcs', + nodeEligibility: [$class: 'AllNodeEligibility'], + triggerIfResult: 'allCases']]} options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '2')) } - parameters{ - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - defaultSlaves: ['built-in'], name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases' - } stages { -- GitLab From 6d0eb3acb0163cadc953a369c1f61e8244c10eae Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:52:59 +0000 Subject: [PATCH 237/447] removed double bracket --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ce16ef46e..7d6fda8bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,11 +4,11 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{[[$class: 'NodeParameterDefinition', + parameters{[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases']]} + triggerIfResult: 'allCases']} options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) -- GitLab From b6f721241abd5f2b29092425dbd165a51cb71552 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 17:54:04 +0000 Subject: [PATCH 238/447] not sure how to hadd class --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d6fda8bc..e6e92cb0f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,11 +4,11 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{[$class: 'NodeParameterDefinition', + parameters{$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases']} + triggerIfResult: 'allCases'} options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) -- GitLab From a94f2ce3575e03ffea846304c13b662d18021e19 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:02:19 +0000 Subject: [PATCH 239/447] still can not get syntax for parameters --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e6e92cb0f..4586a7cca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,11 +4,12 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{$class: 'NodeParameterDefinition', + parameters: [[ $class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases'} + triggerIfResult: 'allCases' ]] + options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) -- GitLab From 9ccf7bc0e707ba3bd4eaa1eea421dd38c81fd8b0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:04:41 +0000 Subject: [PATCH 240/447] still not finding class --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4586a7cca..fe562d8cd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,11 +4,11 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters: [[ $class: 'NodeParameterDefinition', + parameters{ $class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases' ]] + triggerIfResult: 'allCases'} options { disableConcurrentBuilds(abortPrevious: true) -- GitLab From f4edccea553156bf0252ab241987d52803170a15 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:08:41 +0000 Subject: [PATCH 241/447] just removed class line --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe562d8cd..c600591ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{ $class: 'NodeParameterDefinition', + parameters{ allowedSlaves: ['Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'rdhpcs', nodeEligibility: [$class: 'AllNodeEligibility'], -- GitLab From bd4d49c00caca2049a8bf2eda0aba5b3c54a3cf5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:21:31 +0000 Subject: [PATCH 242/447] moved properties into stage --- Jenkinsfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c600591ad..c25ea9ee5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,12 +4,6 @@ def HOMEgfs = 'initial value' pipeline { agent { label 'built-in' } - parameters{ - allowedSlaves: ['Hera-EMC', 'Orion-EMC'], - defaultSlaves: ['built-in'], name: 'rdhpcs', - nodeEligibility: [$class: 'AllNodeEligibility'], - triggerIfResult: 'allCases'} - options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) @@ -73,6 +67,7 @@ pipeline { //when { // expression { MACHINE != 'none' } //} + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) matrix { agent { label "${machine}-emc" } axes { -- GitLab From 45a8a4d014abcdc26228dfe295dab86ba0e7be99 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:23:52 +0000 Subject: [PATCH 243/447] moved properties into script in Run Seection --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c25ea9ee5..60f3b9a1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -67,7 +67,6 @@ pipeline { //when { // expression { MACHINE != 'none' } //} - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) matrix { agent { label "${machine}-emc" } axes { @@ -92,6 +91,7 @@ pipeline { steps { ws(HOMEgfs) { script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 5d29ceb49b8dd6984b148de7621bd7f7ea8f1c31 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:37:16 +0000 Subject: [PATCH 244/447] had machine instead of MACIHNE in node name --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 60f3b9a1c..8ac43a2bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { } stage('Build') { - agent { label "${machine}-emc" } + agent { label "${MACHINE}-emc" } //when { // expression { MACHINE != 'none' } //} @@ -68,7 +68,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "${machine}-emc" } + agent { label "${MACHINE}-emc" } axes { axis { name "Case" -- GitLab From 734d4fc663dce6a99de0967f2bca9ed48d906eb0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 18:57:18 +0000 Subject: [PATCH 245/447] added properties for epic nodes --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ac43a2bb..94c5c9ea8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,7 @@ pipeline { steps { ws(HOMEgfs) { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 39529e5f43927c7f9ce83dc5d01092f4ab5a7763 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 20:08:06 +0000 Subject: [PATCH 246/447] needed to remove RUNTESTS dir on rebuild --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 94c5c9ea8..af08e1d88 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { MACHINE = 'hercules' } } + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) } } } @@ -38,7 +39,6 @@ pipeline { //} steps { script { - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm @@ -53,10 +53,11 @@ pipeline { else { //sh( script: "sorc/build_all.sh -gu", returnStatus: false) sh( script: "sorc/build_all_stub.sh" ) - sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false) + sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true) + sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true) + sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) - sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false) pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } -- GitLab From 9d5ab0448860bbc518601a22a71050488a78e6e6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 20:17:40 +0000 Subject: [PATCH 247/447] added exit 1 to run stub --- Jenkinsfile | 2 +- ci/scripts/run-check_ci_stub.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index af08e1d88..713d7577f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -96,7 +96,7 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: true) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index 01975f566..d98b87db2 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -5,4 +5,5 @@ set -eu HOMEgfs=$1 pslot=$2 -echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log \ No newline at end of file +echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log +exit 1 \ No newline at end of file -- GitLab From 4da660f6cea717d311cf53a0f358d94b84f93d30 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 20:29:23 +0000 Subject: [PATCH 248/447] removed returnstatus flag on run stub --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 713d7577f..17cde6781 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -96,7 +96,7 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: true) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From 92061568edf303796ef60c7ce47b650b3869cd12 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 20:48:24 +0000 Subject: [PATCH 249/447] Updated properties to my controler --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 17cde6781..eb77f601b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,8 @@ pipeline { steps { ws(HOMEgfs) { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 8715f5701c7fddf6c73bdd7a644efc7d411e4be7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:00:20 +0000 Subject: [PATCH 250/447] Can not get parameteres to sork again --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eb77f601b..16c1260e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,8 +92,8 @@ pipeline { steps { ws(HOMEgfs) { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 859d7cb568a4307727b8615e87320a33695d4aca Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:05:32 +0000 Subject: [PATCH 251/447] added defaut slave as running one --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 16c1260e7..5d4234f61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,7 +93,7 @@ pipeline { ws(HOMEgfs) { script { //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 450bcab37afe615bef9b21957068a2ad96816498 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:07:57 +0000 Subject: [PATCH 252/447] added prop in create --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 5d4234f61..aa71cedd4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,6 +82,7 @@ pipeline { steps { ws(HOMEgfs) { script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) env.RUNTESTS = "${HOMEgfs}/RUNTESTS" sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) } -- GitLab From fbbe569b14963a40beb1e32fe43a0bbac6d034d2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:16:55 +0000 Subject: [PATCH 253/447] added prop in create still brokek --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index aa71cedd4..8951bdd9c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -94,7 +94,7 @@ pipeline { ws(HOMEgfs) { script { //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From a2226e2548b609fb77fc7e787069a80776ac6b9a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:21:07 +0000 Subject: [PATCH 254/447] still broken --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8951bdd9c..b9e18372f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,6 +15,8 @@ pipeline { stage('Get Machine') { agent { label 'built-in' } steps { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + script { MACHINE = 'none' for (label in pullRequest.labels) { @@ -82,7 +84,6 @@ pipeline { steps { ws(HOMEgfs) { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) env.RUNTESTS = "${HOMEgfs}/RUNTESTS" sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) } @@ -93,8 +94,6 @@ pipeline { steps { ws(HOMEgfs) { script { - //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 48a34d022915b5bc3c0a2f92d70075372561c52e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:23:34 +0000 Subject: [PATCH 255/447] moved prop under get machine script --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b9e18372f..d044503db 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,9 +15,8 @@ pipeline { stage('Get Machine') { agent { label 'built-in' } steps { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) - script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) MACHINE = 'none' for (label in pullRequest.labels) { echo "Label: ${label}" -- GitLab From 2b7e941b1f7f713151deead6cd5c95bd1a541ca7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 21:56:53 +0000 Subject: [PATCH 256/447] made machine global --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d044503db..df92e7d53 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,5 @@ def MACHINE = 'none' +def machine = 'none' def HOMEgfs = 'initial value' pipeline { @@ -16,7 +17,6 @@ pipeline { agent { label 'built-in' } steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) MACHINE = 'none' for (label in pullRequest.labels) { echo "Label: ${label}" @@ -40,6 +40,7 @@ pipeline { //} steps { script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm -- GitLab From 38ad4b4c2fec5d2c93e28ffc038d2845f7c27c24 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 22:00:01 +0000 Subject: [PATCH 257/447] Label issue --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index df92e7d53..fc0c9fcc3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -115,7 +115,6 @@ pipeline { if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) { pullRequest.removeLabel("CI-${machine}-Running") } - pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") @@ -127,7 +126,6 @@ pipeline { if(pullRequest.labels.contains("CI-${machine}-Running")) { pullRequest.removeLabel("CI-${machine}-Running") } - pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") -- GitLab From 9e5dacf57102fa65bd4f74cd5d18c63818abbe04 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 22:04:23 +0000 Subject: [PATCH 258/447] removed Configuiration class --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fc0c9fcc3..9fb7079f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm -- GitLab From 2ef5a62152dc0e68dade518a1a81cbdfd8150370 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 22:40:02 +0000 Subject: [PATCH 259/447] added stub logic --- ci/scripts/run-check_ci_stub.sh | 4 +++- ci/scripts/utils/ci_utils.sh | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index d98b87db2..48cf160d1 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -6,4 +6,6 @@ HOMEgfs=$1 pslot=$2 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log -exit 1 \ No newline at end of file +if [[ ${pslot} ~= "C48_S2SW" ]]; then + exit 1 +fi \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index a0b45d5d5..0e12a814a 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -88,6 +88,8 @@ function get_pslot () { function create_experiment () { + exit 0 + local yaml_config="${1}" cd "${HOMEgfs}" || exit 1 pr_sha=$(git rev-parse --short HEAD) -- GitLab From a2cef598627d89240439515325a36639111c77da Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 22:48:08 +0000 Subject: [PATCH 260/447] remove exit 0 on create exper wrapper because once worked --- ci/scripts/utils/ci_utils.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 0e12a814a..a0b45d5d5 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -88,8 +88,6 @@ function get_pslot () { function create_experiment () { - exit 0 - local yaml_config="${1}" cd "${HOMEgfs}" || exit 1 pr_sha=$(git rev-parse --short HEAD) -- GitLab From 88799463b258a2cc0c3897cf759bae6fd6a5673c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 19 Jan 2024 23:57:37 +0000 Subject: [PATCH 261/447] two orion fake local adds --- modulefiles/module_gwsetup.orion.lua | 17 +++++------------ ush/module-setup.sh | 7 ++----- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index 93a59c8e5..984cf19b5 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -3,19 +3,12 @@ Load environment to run GFS workflow ci scripts on Orion ]]) prepend_path("MODULEPATH", "/apps/modulefiles/core") -load(pathJoin("contrib","0.1")) -load(pathJoin("rocoto","1.3.3")) -load(pathJoin("git","2.28.0")) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +-- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" -local python_ver=os.getenv("python_ver") or "3.10.8" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) -load("py-jinja2") -load("py-pyyaml") -load("py-numpy") +load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj") +load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4") +load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip") whatis("Description: GFS run setup environment") diff --git a/ush/module-setup.sh b/ush/module-setup.sh index fd656966b..97df183a9 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -30,11 +30,8 @@ elif [[ ${MACHINE_ID} = hercules* ]] ; then set -u elif [[ ${MACHINE_ID} = orion* ]] ; then - # We are on Orion - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib + # We are on Orion TMcG + source $(spack location -i lmod)/lmod/lmod/init/bash set +u module reset set -u -- GitLab From bf18bc80b6bc91e317b8b759b361fc7dc81d6ac3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 00:11:50 +0000 Subject: [PATCH 262/447] added my spak python 3 --- modulefiles/module_gwsetup.orion.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index 984cf19b5..03780b750 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -7,6 +7,7 @@ prepend_path("MODULEPATH", "/apps/modulefiles/core") -- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake") +load ("python-3.9.12-gcc-7.3.1-cfrtozo") load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj") load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4") load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip") -- GitLab From 26a3512778bba9476ff7d9e66c11bcbf66dd28d2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 00:20:45 +0000 Subject: [PATCH 263/447] got =~ backwards --- ci/scripts/run-check_ci_stub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index 48cf160d1..e01531d68 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -6,6 +6,6 @@ HOMEgfs=$1 pslot=$2 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log -if [[ ${pslot} ~= "C48_S2SW" ]]; then +if [[ ${pslot} =~ "C48_S2SW" ]]; then exit 1 fi \ No newline at end of file -- GitLab From 77af83d336680a2d6361489615f37ea82065e3b3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 01:32:52 +0000 Subject: [PATCH 264/447] explitly remove existing experment before re-running create --- ci/scripts/utils/ci_utils.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index a0b45d5d5..22f4d496f 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -92,11 +92,17 @@ function create_experiment () { cd "${HOMEgfs}" || exit 1 pr_sha=$(git rev-parse --short HEAD) case=$(basename "${yaml_config}" .yaml) || true + export pslot=${case}_${pr_sha} source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - export pslot=${case}_${pr_sha} + if [[ -d "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" ]]; then + echo "Removing existing experiment directory: ${HOMEgfs}/RUNDIRS/EXPDIR/${case}_${pr_sha}" + rm -rf "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" + rm -rf "${HOMEgfs}/RUNDIRS/COMROT/${pslot}" + fi + "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } -- GitLab From b6ffee942da7afc244643a5da0232f25b77486f3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 01:38:41 +0000 Subject: [PATCH 265/447] hard coded HOMEgfs for testing --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9fb7079f3..875709bda 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ def MACHINE = 'none' def machine = 'none' -def HOMEgfs = 'initial value' +def HOMEgfs = '/home/tmcguinness/Jenkins/Hera/workspace/Global_PR-218' pipeline { agent { label 'built-in' } @@ -34,7 +34,7 @@ pipeline { } stage('Build') { - agent { label "${MACHINE}-emc" } + agent { label "orion-emc" } //when { // expression { MACHINE != 'none' } //} @@ -71,7 +71,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "${MACHINE}-emc" } + agent { label "orion-emc" } axes { axis { name "Case" -- GitLab From 7c7fbbe324a95bcae4b2064772ee1d19cc411649 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 01:41:31 +0000 Subject: [PATCH 266/447] testing on node hera not orion --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 875709bda..eae8ec2ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,7 +34,7 @@ pipeline { } stage('Build') { - agent { label "orion-emc" } + agent { label "hera-emc" } //when { // expression { MACHINE != 'none' } //} @@ -71,7 +71,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "orion-emc" } + agent { label "hera-emc" } axes { axis { name "Case" -- GitLab From 0638b6df0667aa18ffeaa8dd6a36baba5dd8dc03 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 01:52:06 +0000 Subject: [PATCH 267/447] dump remove label constraint for now --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eae8ec2ad..29c1036fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ pipeline { steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - pullRequest.removeLabel("CI-${machine}-Ready") + //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm HOMEgfs = "${WORKSPACE}" @@ -60,7 +60,7 @@ pipeline { sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) - pullRequest.removeLabel("CI-${machine}-Building") + //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } } -- GitLab From 6fc3581fe20beb24ec91a71fef6f99361763e90b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 20 Jan 2024 02:14:04 +0000 Subject: [PATCH 268/447] RUNTESTS was wrong and yes works --- ci/scripts/utils/ci_utils.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 22f4d496f..9c9c5780e 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -97,12 +97,8 @@ function create_experiment () { source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - if [[ -d "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" ]]; then - echo "Removing existing experiment directory: ${HOMEgfs}/RUNDIRS/EXPDIR/${case}_${pr_sha}" - rm -rf "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" - rm -rf "${HOMEgfs}/RUNDIRS/COMROT/${pslot}" - fi + set -eux - "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + yes | ${HOMEgfs}/workflow/create_experiment.py --yaml "${yaml_config}" } -- GitLab From 4e807cc2a8fc349c2f701980ee0e8f394343686f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 15:29:10 +0000 Subject: [PATCH 269/447] removed hardcoded machine label on nodes --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 29c1036fa..caa17fc1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ def MACHINE = 'none' def machine = 'none' -def HOMEgfs = '/home/tmcguinness/Jenkins/Hera/workspace/Global_PR-218' +def HOMEgfs = 'none' pipeline { agent { label 'built-in' } @@ -34,7 +34,7 @@ pipeline { } stage('Build') { - agent { label "hera-emc" } + agent { label "${MACHINE}-emc" } //when { // expression { MACHINE != 'none' } //} @@ -71,7 +71,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "hera-emc" } + agent { label "${machine}-emc" } axes { axis { name "Case" -- GitLab From b66b9bcaf77fc6368084c0695b0bf246e8765472 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 15:40:17 +0000 Subject: [PATCH 270/447] restore away from local build cludges --- modulefiles/module_gwsetup.orion.lua | 18 ++++++++++++------ ush/module-setup.sh | 7 +++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index 03780b750..93a59c8e5 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -3,13 +3,19 @@ Load environment to run GFS workflow ci scripts on Orion ]]) prepend_path("MODULEPATH", "/apps/modulefiles/core") +load(pathJoin("contrib","0.1")) +load(pathJoin("rocoto","1.3.3")) +load(pathJoin("git","2.28.0")) --- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -load ("python-3.9.12-gcc-7.3.1-cfrtozo") -load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj") -load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4") -load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip") +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") whatis("Description: GFS run setup environment") diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 97df183a9..fd656966b 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -30,8 +30,11 @@ elif [[ ${MACHINE_ID} = hercules* ]] ; then set -u elif [[ ${MACHINE_ID} = orion* ]] ; then - # We are on Orion TMcG - source $(spack location -i lmod)/lmod/lmod/init/bash + # We are on Orion + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib set +u module reset set -u -- GitLab From 29d790cb7a23f607591a9d9458b9db52eab37eca Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 15:53:24 +0000 Subject: [PATCH 271/447] some linting --- ci/scripts/run-check_ci_stub.sh | 2 +- ci/scripts/utils/ci_utils.sh | 2 +- sorc/build_all_stub.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index e01531d68..b58664992 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -5,7 +5,7 @@ set -eu HOMEgfs=$1 pslot=$2 -echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log +echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log" if [[ ${pslot} =~ "C48_S2SW" ]]; then exit 1 fi \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 9c9c5780e..600778db6 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -99,6 +99,6 @@ function create_experiment () { set -eux - yes | ${HOMEgfs}/workflow/create_experiment.py --yaml "${yaml_config}" + yes || true | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh index 9acf802c4..a5ddf98d1 100755 --- a/sorc/build_all_stub.sh +++ b/sorc/build_all_stub.sh @@ -2,4 +2,4 @@ set -eu -echo "Ran build all stub in ${PWD}" > BUILD_STUB_RAN_$(date +%d%H%M) \ No newline at end of file +echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)" \ No newline at end of file -- GitLab From 1b47f78c22acdb6475284b449722c49e4475a3d8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 15:54:37 +0000 Subject: [PATCH 272/447] some eols --- ci/scripts/run-check_ci_stub.sh | 2 +- sorc/build_all_stub.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh index b58664992..23d3da837 100755 --- a/ci/scripts/run-check_ci_stub.sh +++ b/ci/scripts/run-check_ci_stub.sh @@ -8,4 +8,4 @@ pslot=$2 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log" if [[ ${pslot} =~ "C48_S2SW" ]]; then exit 1 -fi \ No newline at end of file +fi diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh index a5ddf98d1..6221b6bb5 100755 --- a/sorc/build_all_stub.sh +++ b/sorc/build_all_stub.sh @@ -2,4 +2,4 @@ set -eu -echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)" \ No newline at end of file +echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)" -- GitLab From a30a5553c60d8047f77b6b0960bd68a2f5ade7f4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 16:21:41 +0000 Subject: [PATCH 273/447] wrong MACHINE in node label --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index caa17fc1c..4832e1aee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -71,7 +71,7 @@ pipeline { // expression { MACHINE != 'none' } //} matrix { - agent { label "${machine}-emc" } + agent { label "${MACHINE}-emc" } axes { axis { name "Case" -- GitLab From ad1c71f7bffc43fe33055050de6d5fc9e8bd9b08 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 16:34:47 +0000 Subject: [PATCH 274/447] removed y pipe and added rm case in pipeline --- Jenkinsfile | 1 + ci/scripts/utils/ci_utils.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4832e1aee..b7be83407 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,6 +85,7 @@ pipeline { ws(HOMEgfs) { script { env.RUNTESTS = "${HOMEgfs}/RUNTESTS" + sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" ) sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) } } diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 600778db6..a92ce66dc 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -99,6 +99,6 @@ function create_experiment () { set -eux - yes || true | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" } -- GitLab From 65cc185e4b45fa22f190e45f6aa987f8e722bf84 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 16:39:56 +0000 Subject: [PATCH 275/447] removed DefaultSlave in parameters --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7be83407..dc8d01e61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm -- GitLab From 30dea74c6657400bd406f33a54c0c9e5aa0c95d6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 17:09:24 +0000 Subject: [PATCH 276/447] added rm COMROOT line in pipeline --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc8d01e61..0d7fa4fe0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm @@ -86,7 +86,8 @@ pipeline { script { env.RUNTESTS = "${HOMEgfs}/RUNTESTS" sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" ) - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false) + sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/COMROOT/${Case}_*" ) + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } -- GitLab From 8657be527cf7401d7fca63fcddcc0cd9e707d2a1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 17:33:55 +0000 Subject: [PATCH 277/447] try var in param --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d7fa4fe0..bd008aa85 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['${machine}-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm -- GitLab From 995b437cf1a768da928a85c2ac45485b17176efd Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 17:43:50 +0000 Subject: [PATCH 278/447] trying to add more choices --- Jenkinsfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd008aa85..42e29d474 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,8 @@ pipeline { //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['${machine}-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'], + [choice(choices: ['C48_ATM', 'C48_S2SW', 'C96_atm3DVar', 'C48_S2SWA_gefs'], name: 'Cases')] ])]) //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm @@ -53,8 +54,8 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - //sh( script: "sorc/build_all.sh -gu", returnStatus: false) - sh( script: "sorc/build_all_stub.sh" ) + sh( script: "sorc/build_all.sh -gu", returnStatus: false) + //sh( script: "sorc/build_all_stub.sh" ) sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true) sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true) sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true) @@ -98,8 +99,8 @@ pipeline { script { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From c2a9c865d497601e68d1a1f78b6b74468df228ab Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 17:48:38 +0000 Subject: [PATCH 279/447] add back real build run --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 42e29d474..17b449886 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,9 +35,9 @@ pipeline { stage('Build') { agent { label "${MACHINE}-emc" } - //when { - // expression { MACHINE != 'none' } - //} + when { + expression { MACHINE != 'none' } + } steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'], -- GitLab From 49a7f716e7ac68577871a011577fb4fee62c1c3b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 18:09:15 +0000 Subject: [PATCH 280/447] revert --- Jenkinsfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 17b449886..cfc820c0f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,13 +35,12 @@ pipeline { stage('Build') { agent { label "${MACHINE}-emc" } - when { - expression { MACHINE != 'none' } - } + //when { + // expression { MACHINE != 'none' } + //} steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'], - [choice(choices: ['C48_ATM', 'C48_S2SW', 'C96_atm3DVar', 'C48_S2SWA_gefs'], name: 'Cases')] ])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) //pullRequest.removeLabel("CI-${machine}-Ready") pullRequest.addLabel("CI-${machine}-Building") checkout scm -- GitLab From bfb1ef49edc20d9ba6cda37650276d69d8771fcb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:15:57 +0000 Subject: [PATCH 281/447] ready to start testing stub pipeline with system build --- Jenkinsfile | 57 ++++++++++++++++++++---------------- ci/scripts/utils/ci_utils.sh | 4 +-- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cfc820c0f..707170368 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,6 +18,7 @@ pipeline { steps { script { MACHINE = 'none' + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { @@ -29,37 +30,42 @@ pipeline { } } machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + HOME = "${WORKSPACE}" } } } - - stage('Build') { + matrix { agent { label "${MACHINE}-emc" } - //when { - // expression { MACHINE != 'none' } - //} + axes { + axes { name "system" + values "gfs", "gefs"} + } + } + stage('Build') { steps { + ws( "${HOME}/${system}") script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - //pullRequest.removeLabel("CI-${machine}-Ready") - pullRequest.addLabel("CI-${machine}-Building") checkout scm - HOMEgfs = "${WORKSPACE}" - } - script { + HOMEgfs = "${HOME}/${system}" env.MACHINE_ID = MACHINE - if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) { + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - sh( script: "sorc/build_all.sh -gu", returnStatus: false) - //sh( script: "sorc/build_all_stub.sh" ) - sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true) - sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true) - sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true) + if (system == "gfs") { + //sh( script: "sorc/build_all.sh -gu", returnStatus: false) + sh( script: "sorc/build_all_stub.sh", returnStatus: false) + } + else if (system == "gefs") { + //sh( script: "sorc/build_all.sh -guw", returnStatus: false) + sh( script: "sorc/build_all_stub.sh", returnStatus: false) + } + sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } @@ -75,21 +81,20 @@ pipeline { axes { axis { name "Case" - //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_ATM", "C48_S2SW" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } stages { stage('Create Experiment') { steps { - ws(HOMEgfs) { script { - env.RUNTESTS = "${HOMEgfs}/RUNTESTS" - sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" ) - sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/COMROOT/${Case}_*" ) + HOMEgfs = "${HOME}/gfs" + env.RUNTESTS = "${HOME}/RUNTESTS" + env.HOME = HOME + sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" ) + sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" ) sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } - } } } stage('Run Experiments') { @@ -98,8 +103,8 @@ pipeline { script { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") + //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index a92ce66dc..f71221ccc 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -97,8 +97,8 @@ function create_experiment () { source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - set -eux + system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true - "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}" + "${HOME}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}" } -- GitLab From a2e8f076d9d017020fd0d9c138a5854192f9c034 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:18:44 +0000 Subject: [PATCH 282/447] wrap a stages around build matrix stage --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 707170368..159edf889 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,6 +41,7 @@ pipeline { values "gfs", "gefs"} } } + stages { stage('Build') { steps { ws( "${HOME}/${system}") @@ -71,6 +72,7 @@ pipeline { } } } + } stage('Run Tests') { //when { -- GitLab From d1e23592349144d5c858a6d2fd33abe1986b7460 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:22:25 +0000 Subject: [PATCH 283/447] wrap matrix around build stage --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 159edf889..d78792ab3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,8 +40,6 @@ pipeline { axes { name "system" values "gfs", "gefs"} } - } - stages { stage('Build') { steps { ws( "${HOME}/${system}") -- GitLab From 54baf54a4b2ac642282370cd55927d9f5b930fd5 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:30:04 +0000 Subject: [PATCH 284/447] needed axis not two axes --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d78792ab3..9dede9d82 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { matrix { agent { label "${MACHINE}-emc" } axes { - axes { name "system" + axis { name "system" values "gfs", "gefs"} } stage('Build') { -- GitLab From 04288ce646ce5f6c659fb2c0b3db3046451972da Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:34:31 +0000 Subject: [PATCH 285/447] put matrix wrap in buid stage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9dede9d82..3870095aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,13 +34,13 @@ pipeline { } } } + stage('Build') { matrix { agent { label "${MACHINE}-emc" } axes { axis { name "system" values "gfs", "gefs"} } - stage('Build') { steps { ws( "${HOME}/${system}") script { -- GitLab From b8b8070b11aa6404dd03225f40c55f5704779d63 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:38:01 +0000 Subject: [PATCH 286/447] still trying to get matrix hierachey to work --- Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3870095aa..24d77c29b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,9 @@ pipeline { axes { axis { name "system" values "gfs", "gefs"} - } + } + stages { + stage("build ${system}") { steps { ws( "${HOME}/${system}") script { @@ -69,6 +71,8 @@ pipeline { pullRequest.addLabel("CI-${machine}-Running") } } + } + } } } -- GitLab From dcf7a87c50249540b70047696056232b70a3f36e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:39:29 +0000 Subject: [PATCH 287/447] system literal literal --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 24d77c29b..7e66888a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { values "gfs", "gefs"} } stages { - stage("build ${system}") { + stage("build system") { steps { ws( "${HOME}/${system}") script { -- GitLab From 686be118a12284f0b564df389cd505cf485f5ce4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 20:52:18 +0000 Subject: [PATCH 288/447] ws needed body --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7e66888a9..dd5dc9de8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,7 @@ pipeline { stages { stage("build system") { steps { - ws( "${HOME}/${system}") + ws( "${HOME}/${system}") { script { checkout scm HOMEgfs = "${HOME}/${system}" @@ -70,6 +70,7 @@ pipeline { //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } + } } } } -- GitLab From 81380a850e5491a14dd354f06f7e3761f02c1ae8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 22:06:49 +0000 Subject: [PATCH 289/447] remove workspaces --- Jenkinsfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd5dc9de8..0a6cbe666 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,13 +44,14 @@ pipeline { stages { stage("build system") { steps { - ws( "${HOME}/${system}") { script { + HOMEgfs = "${HOME}/HOMEgfs/${system}" + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) checkout scm - HOMEgfs = "${HOME}/${system}" env.MACHINE_ID = MACHINE - if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { - HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim() + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { @@ -70,7 +71,6 @@ pipeline { //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } - } } } } @@ -104,15 +104,14 @@ pipeline { } stage('Run Experiments') { steps { - ws(HOMEgfs) { script { + HOMEgfs = "${HOME}/gfs" pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } - } } } } -- GitLab From e0a420fca038fb86d3bb5221d01c1dde8484b94b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 22:53:07 +0000 Subject: [PATCH 290/447] WORKSPACE git just outside of build matrix in node --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0a6cbe666..b2f727535 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,13 +30,13 @@ pipeline { } } machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - HOME = "${WORKSPACE}" } } } stage('Build') { matrix { agent { label "${MACHINE}-emc" } + script { TESTDIR = "${WORKSPACE}/TESTDIR" } axes { axis { name "system" values "gfs", "gefs"} @@ -45,7 +45,7 @@ pipeline { stage("build system") { steps { script { - HOMEgfs = "${HOME}/HOMEgfs/${system}" + HOMEgfs = "${HOME}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) checkout scm -- GitLab From d9303a2f54746dd812f20fad1c444826cf5a0e86 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 22:59:30 +0000 Subject: [PATCH 291/447] Added TESTDIR --- Jenkinsfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b2f727535..82d0e1405 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,6 @@ pipeline { steps { script { MACHINE = 'none' - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { @@ -45,7 +44,9 @@ pipeline { stage("build system") { steps { script { - HOMEgfs = "${HOME}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + TESTDIR = "${WORKSPACE}/TESTDIR" + HOMEgfs = "${TESTDIR}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) checkout scm @@ -66,7 +67,7 @@ pipeline { sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true) //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") @@ -93,9 +94,8 @@ pipeline { stage('Create Experiment') { steps { script { - HOMEgfs = "${HOME}/gfs" - env.RUNTESTS = "${HOME}/RUNTESTS" - env.HOME = HOME + env.RUNTESTS = "${TESTDIR}/RUNTESTS" + env.HOME = HOMEgfs sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" ) sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" ) sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) @@ -105,7 +105,6 @@ pipeline { stage('Run Experiments') { steps { script { - HOMEgfs = "${HOME}/gfs" pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) -- GitLab From 9b28ba32cfb8c543730655e0e6d517fd98be7965 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 23:00:58 +0000 Subject: [PATCH 292/447] deleted old script line --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 82d0e1405..a6491ec4a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,6 @@ pipeline { stage('Build') { matrix { agent { label "${MACHINE}-emc" } - script { TESTDIR = "${WORKSPACE}/TESTDIR" } axes { axis { name "system" values "gfs", "gefs"} -- GitLab From a6e08390e9a058ef91667633852f5473aed97a71 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 23:06:10 +0000 Subject: [PATCH 293/447] dir step needed bodey --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6491ec4a..5e4fc1293 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,7 +47,7 @@ pipeline { TESTDIR = "${WORKSPACE}/TESTDIR" HOMEgfs = "${TESTDIR}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) + dir(HOMEgfs) { checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { @@ -70,6 +70,7 @@ pipeline { //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") + } } } } -- GitLab From 6687bd54623745035cb3b2a7a58253360ae59bba Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 22 Jan 2024 23:27:03 +0000 Subject: [PATCH 294/447] created dedicated stage for getting global workspace for the PR --- Jenkinsfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e4fc1293..e4e165d0d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,7 @@ def MACHINE = 'none' def machine = 'none' def HOMEgfs = 'none' +def TESTDIR = 'none' pipeline { agent { label 'built-in' } @@ -32,6 +33,16 @@ pipeline { } } } + + stage('Get Common Workspace') { + agent { label "${MACHINE}-emc" } + steps ( timeout(time: 1, unit: 'HOURS') ) { + script { + TESTDIR = "${WORKSPACE}/TESTDIR" + } + } + } + stage('Build') { matrix { agent { label "${MACHINE}-emc" } @@ -44,7 +55,6 @@ pipeline { steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - TESTDIR = "${WORKSPACE}/TESTDIR" HOMEgfs = "${TESTDIR}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { -- GitLab From 118d43d2a82c1fab5a5066fc831fdc1022578328 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 00:16:21 +0000 Subject: [PATCH 295/447] made HOMEgfs local in matrix and removed from global --- Jenkinsfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e4e165d0d..6b4e0b43b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,5 @@ def MACHINE = 'none' def machine = 'none' -def HOMEgfs = 'none' def TESTDIR = 'none' pipeline { @@ -54,14 +53,14 @@ pipeline { stage("build system") { steps { script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOMEgfs = "${TESTDIR}/${system}" + def HOMEgfs = "${TESTDIR}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim() + sh( script: "cat sorc/BUILT_sema", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { @@ -73,7 +72,7 @@ pipeline { //sh( script: "sorc/build_all.sh -guw", returnStatus: false) sh( script: "sorc/build_all_stub.sh", returnStatus: false) } - sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true) + sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true) -- GitLab From e601257c6b4a47b8a1d179f996b71863e8066277 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 01:19:08 +0000 Subject: [PATCH 296/447] global HOME TESTDIR --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b4e0b43b..e06e0f967 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,7 @@ def MACHINE = 'none' def machine = 'none' def TESTDIR = 'none' +def HOME = 'none' pipeline { agent { label 'built-in' } @@ -37,7 +38,7 @@ pipeline { agent { label "${MACHINE}-emc" } steps ( timeout(time: 1, unit: 'HOURS') ) { script { - TESTDIR = "${WORKSPACE}/TESTDIR" + HOME = "${WORKSPACE}/TESTDIR" } } } @@ -53,14 +54,14 @@ pipeline { stage("build system") { steps { script { - def HOMEgfs = "${TESTDIR}/${system}" + def HOMEgfs = "${HOME}/${system}" properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_sema", returnStdout: true).trim() + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { @@ -104,6 +105,7 @@ pipeline { steps { script { env.RUNTESTS = "${TESTDIR}/RUNTESTS" + def HOMEgfs = "${HOME}/TESTDIR/gfs" env.HOME = HOMEgfs sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" ) sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" ) -- GitLab From c422f617a88641c16bd361079f3d399384decfd8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 01:52:40 +0000 Subject: [PATCH 297/447] added local HOMEgfs and global HOME in Run Experiments --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e06e0f967..41fc6181b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,5 @@ def MACHINE = 'none' def machine = 'none' -def TESTDIR = 'none' def HOME = 'none' pipeline { @@ -104,8 +103,8 @@ pipeline { stage('Create Experiment') { steps { script { - env.RUNTESTS = "${TESTDIR}/RUNTESTS" - def HOMEgfs = "${HOME}/TESTDIR/gfs" + env.RUNTESTS = "${HOME}/RUNTESTS" + def HOMEgfs = "${HOME}/gfs" env.HOME = HOMEgfs sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" ) sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" ) @@ -116,10 +115,11 @@ pipeline { stage('Run Experiments') { steps { script { - pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim() + def HOMEgfs = "${HOME}/gfs" + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}") + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From 86f2c903b0e1941f06c2beb29f254446656589ec Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 02:05:26 +0000 Subject: [PATCH 298/447] updated TESTDIR to global HOME --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 41fc6181b..9ecf37a7f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,7 +75,7 @@ pipeline { sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) - sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true) + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") -- GitLab From 45601f5be369cf7565b174e541478d54cd3ca3f0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 02:34:27 +0000 Subject: [PATCH 299/447] remved stubs to run on Orion --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9ecf37a7f..8ded06712 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,12 +65,12 @@ pipeline { } else { if (system == "gfs") { - //sh( script: "sorc/build_all.sh -gu", returnStatus: false) - sh( script: "sorc/build_all_stub.sh", returnStatus: false) + sh( script: "sorc/build_all.sh -gu", returnStatus: false) + //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } else if (system == "gefs") { - //sh( script: "sorc/build_all.sh -guw", returnStatus: false) - sh( script: "sorc/build_all_stub.sh", returnStatus: false) + sh( script: "sorc/build_all.sh -guw", returnStatus: false) + //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } @@ -118,8 +118,8 @@ pipeline { def HOMEgfs = "${HOME}/gfs" pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}") + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From 3146a39d1619c18bb587c1b9ae80f8ceceb154fe Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 02:46:53 +0000 Subject: [PATCH 300/447] need updated buid all args for gefs --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ded06712..9f216031c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,7 +69,7 @@ pipeline { //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } else if (system == "gefs") { - sh( script: "sorc/build_all.sh -guw", returnStatus: false) + sh( script: "sorc/build_all.sh -gu -a UFS_app", returnStatus: false) //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) -- GitLab From 343020c9f6393380fc1e350ca7024f437dbd0963 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 04:20:55 +0000 Subject: [PATCH 301/447] use old build all args for both systems --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9f216031c..450608b48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,7 +69,7 @@ pipeline { //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } else if (system == "gefs") { - sh( script: "sorc/build_all.sh -gu -a UFS_app", returnStatus: false) + sh( script: "sorc/build_all.sh -gu", returnStatus: false) //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) -- GitLab From 508e354e6783ac1f11f10f316ab246369e2ffbc0 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 14:33:19 +0000 Subject: [PATCH 302/447] removed stubs add all nodes with restart selector and put back label rm --- Jenkinsfile | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 450608b48..4ffc1dc0c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ pipeline { options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) - buildDiscarder(logRotator(numToKeepStr: '2')) + // buildDiscarder(logRotator(numToKeepStr: '2')) } stages { @@ -54,7 +54,7 @@ pipeline { steps { script { def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { checkout scm @@ -66,18 +66,17 @@ pipeline { else { if (system == "gfs") { sh( script: "sorc/build_all.sh -gu", returnStatus: false) - //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } else if (system == "gefs") { + // TODO: need to add gefs build arguments from a yaml file sh( script: "sorc/build_all.sh -gu", returnStatus: false) - //sh( script: "sorc/build_all_stub.sh", returnStatus: false) } sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } sh( script: "sorc/link_workflow.sh", returnStatus: false) sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work - //pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } } @@ -88,9 +87,6 @@ pipeline { } stage('Run Tests') { - //when { - // expression { MACHINE != 'none' } - //} matrix { agent { label "${MACHINE}-emc" } axes { @@ -119,7 +115,6 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) - //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}") pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } -- GitLab From 2b684c29d90623fd8ae1f76cb1bd749f2193af0e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 14:37:02 +0000 Subject: [PATCH 303/447] remove Running label without check --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ffc1dc0c..d7992ad8f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -128,9 +128,11 @@ pipeline { post { success { script { - if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) { - pullRequest.removeLabel("CI-${machine}-Running") - } + //if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) { + // pullRequest.removeLabel("CI-${machine}-Running") + //} + // TODO: contains mehthod does not work + pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") @@ -139,9 +141,7 @@ pipeline { } failure { script { - if(pullRequest.labels.contains("CI-${machine}-Running")) { - pullRequest.removeLabel("CI-${machine}-Running") - } + pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") -- GitLab From 6775d4988357f0e851c0b515ea4e0338145f8bf8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 20:32:49 +0000 Subject: [PATCH 304/447] add dir into sorc --- Jenkinsfile | 107 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d7992ad8f..225d3530a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,47 +43,61 @@ pipeline { } stage('Build') { - matrix { - agent { label "${MACHINE}-emc" } - axes { - axis { name "system" - values "gfs", "gefs"} - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { - checkout scm - env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + matrix { + agent { label "${MACHINE}-emc" } + axes { + axis { + name "system" + values "gfs", "gefs" } - else { - if (system == "gfs") { - sh( script: "sorc/build_all.sh -gu", returnStatus: false) - } - else if (system == "gefs") { - // TODO: need to add gefs build arguments from a yaml file - sh( script: "sorc/build_all.sh -gu", returnStatus: false) + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) { + checkout scm + env.MACHINE_ID = MACHINE + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } else { + if (system == "gfs") { + dir("sorc") { + sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) + sh( script: "build_all.sh -gu", returnStatus: false) + } + } else if (system == "gefs") { + // TODO: need to add gefs build arguments from a yaml file + dir("sorc") { + sh( script: "echo $PWD", returnStatus: true) + sh( script: "build_all.sh -gu", returnStatus: false) + } + } + sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) + } + sh( script: "sorc/link_workflow.sh", returnStatus: false) + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Running") + } + } } - sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } - sh( script: "sorc/link_workflow.sh", returnStatus: false) - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work + } + script { pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") - } } } + script { + pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.addLabel("CI-${machine}-Running") } - } - } } stage('Run Tests') { @@ -110,18 +124,29 @@ pipeline { } stage('Run Experiments') { steps { - script { - def HOMEgfs = "${HOME}/gfs" - pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") - } + script { + def HOMEgfs = "${HOME}/gfs" + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + try { + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + } catch (Exception e) { + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + archiveArtifacts artifacts: "${line}", fingerprint: true + } + } + } + pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") + error("Failed to run experiments ${Case} on ${machine}") + } } } } } - } + } } -- GitLab From 785608124e0f5bc625bfde82bdb4863adffa353f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 20:51:58 +0000 Subject: [PATCH 305/447] moved mkdir on RUNTESTS into top stage in Create Experments --- Jenkinsfile | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 225d3530a..87f9e3593 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,38 +69,36 @@ pipeline { dir("sorc") { sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) sh( script: "build_all.sh -gu", returnStatus: false) + sh( script: "link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { // TODO: need to add gefs build arguments from a yaml file dir("sorc") { sh( script: "echo $PWD", returnStatus: true) sh( script: "build_all.sh -gu", returnStatus: false) + sh( script: "link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } - sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true) } - sh( script: "sorc/link_workflow.sh", returnStatus: false) - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Running") } } } } } + } + } + + stage('Run Tests') { + steps { script { + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } } - script { - pullRequest.removeLabel("CI-${machine}-Building") - pullRequest.addLabel("CI-${machine}-Running") - } - } - - stage('Run Tests') { matrix { agent { label "${MACHINE}-emc" } axes { -- GitLab From 75bccbd7dd0bc7155795fc3d93a9f8fb37412cb4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 21:17:04 +0000 Subject: [PATCH 306/447] added entire stage to mkdir TESTDIR and update labels --- Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 87f9e3593..be6d3659b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,7 +90,7 @@ pipeline { } } - stage('Run Tests') { + stage('Setup RUNTESTS') { steps { script { sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) @@ -99,6 +99,10 @@ pipeline { pullRequest.addLabel("CI-${machine}-Running") } } + + } + + stage('Run Tests') { matrix { agent { label "${MACHINE}-emc" } axes { -- GitLab From b2b52432b37b44712385c410e4cfe902b71315f8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 21:29:28 +0000 Subject: [PATCH 307/447] added full path for dir() into sorc --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be6d3659b..1193222d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,16 +66,16 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { if (system == "gfs") { - dir("sorc") { - sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) + dir("${HOMEgfs}/sorc") { + sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) sh( script: "build_all.sh -gu", returnStatus: false) sh( script: "link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { // TODO: need to add gefs build arguments from a yaml file - dir("sorc") { - sh( script: "echo $PWD", returnStatus: true) + dir("${HOMEgfs}/sorc") { + sh( script: "echo 'In gefs block in dir $PWD'", returnStatus: true) sh( script: "build_all.sh -gu", returnStatus: false) sh( script: "link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) -- GitLab From 5965d86b2ba04346c7b2811b693d3a67b47c129e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 21:30:14 +0000 Subject: [PATCH 308/447] reduced save to three --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1193222d6..c20506b96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ pipeline { options { disableConcurrentBuilds(abortPrevious: true) skipDefaultCheckout(true) - // buildDiscarder(logRotator(numToKeepStr: '2')) + buildDiscarder(logRotator(numToKeepStr: '3')) } stages { -- GitLab From e22dffaa134005b8de22cc93eb9b885897c8bd7e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 22:05:27 +0000 Subject: [PATCH 309/447] can not figure out why dir() is not working --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c20506b96..a9263a32a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,6 +66,8 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { if (system == "gfs") { + sh( script: "echo 'What is going on here: ${HOMEgfs}/sorc'", returnStatus: true) + sh( script: "ls -lrtha ${HOMEgfs}/sorc'", returnStatus: true) dir("${HOMEgfs}/sorc") { sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) sh( script: "build_all.sh -gu", returnStatus: false) -- GitLab From b394c3a520dea2964c67bf50bcc78cd7889d4389 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 23 Jan 2024 22:31:34 +0000 Subject: [PATCH 310/447] added a few more degubs and . for pwd path --- Jenkinsfile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9263a32a..532b98a47 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,20 +66,21 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { if (system == "gfs") { - sh( script: "echo 'What is going on here: ${HOMEgfs}/sorc'", returnStatus: true) - sh( script: "ls -lrtha ${HOMEgfs}/sorc'", returnStatus: true) + sh( script: "echo 'What is going on here in gfs block: ${HOMEgfs}/sorc'", returnStatus: true) + sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true) dir("${HOMEgfs}/sorc") { - sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) - sh( script: "build_all.sh -gu", returnStatus: false) - sh( script: "link_workflow.sh", returnStatus: false) + sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) + sh( script: "./build_all.sh -gu", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { + sh( script: "echo 'What is going on here in gefs: ${HOMEgfs}/sorc'", returnStatus: true) // TODO: need to add gefs build arguments from a yaml file dir("${HOMEgfs}/sorc") { - sh( script: "echo 'In gefs block in dir $PWD'", returnStatus: true) - sh( script: "build_all.sh -gu", returnStatus: false) - sh( script: "link_workflow.sh", returnStatus: false) + sh( script: "echo 'In gefs block in dir $PWD';ls -l", returnStatus: true) + sh( script: "./build_all.sh -gu", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } -- GitLab From 9b02f36849283c0be0bf2a5bed73cd6777d4881d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 25 Jan 2024 16:01:54 +0000 Subject: [PATCH 311/447] added an explicit cd into the gfs/sorc dir to run build_all --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 532b98a47..7fc00f355 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,7 @@ pipeline { sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true) dir("${HOMEgfs}/sorc") { sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) - sh( script: "./build_all.sh -gu", returnStatus: false) + sh( script: "cd ${HOMEgfs}/sorc;echo "PWD:${PWD};./build_all.sh -gu", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From c7053d14e3ce13ee6a4b0fa4393c0b1d8d1a1809 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 25 Jan 2024 16:09:46 +0000 Subject: [PATCH 312/447] fixed quote on echo PWD on script sh command line --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7fc00f355..efa9cbb56 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,7 @@ pipeline { sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true) dir("${HOMEgfs}/sorc") { sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) - sh( script: "cd ${HOMEgfs}/sorc;echo "PWD:${PWD};./build_all.sh -gu", returnStatus: false) + sh( script: "cd ${HOMEgfs}/sorc;echo 'PWD:${PWD}';./build_all.sh -gu", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From 457b3d7b7e818e73a4bdd53edb558803e05df02d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 25 Jan 2024 19:35:31 +0000 Subject: [PATCH 313/447] added explicit git command for updating submoules recursivly, scm was apearing to skip gdas submodules --- Jenkinsfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index efa9cbb56..3474d8136 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,20 +65,16 @@ pipeline { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { + sh( script: "git submodule update --init --recursive", returnStatus: true) if (system == "gfs") { - sh( script: "echo 'What is going on here in gfs block: ${HOMEgfs}/sorc'", returnStatus: true) - sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true) dir("${HOMEgfs}/sorc") { - sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) - sh( script: "cd ${HOMEgfs}/sorc;echo 'PWD:${PWD}';./build_all.sh -gu", returnStatus: false) + sh( script: "./build_all.sh -gu", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { - sh( script: "echo 'What is going on here in gefs: ${HOMEgfs}/sorc'", returnStatus: true) // TODO: need to add gefs build arguments from a yaml file dir("${HOMEgfs}/sorc") { - sh( script: "echo 'In gefs block in dir $PWD';ls -l", returnStatus: true) sh( script: "./build_all.sh -gu", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) -- GitLab From 6acc9180c5b1ec9950f794b28460a939f3399d86 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 26 Jan 2024 16:32:46 +0000 Subject: [PATCH 314/447] added max jobs on build all to 4 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3474d8136..530ce7d16 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,7 +68,7 @@ pipeline { sh( script: "git submodule update --init --recursive", returnStatus: true) if (system == "gfs") { dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu", returnStatus: false) + sh( script: "./build_all.sh -gu -j 4", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From 258e9c7b4d9debacbca348e521b549b7f6f51d2d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 26 Jan 2024 22:21:50 +0000 Subject: [PATCH 315/447] updated pipline with node on setup RUNDIR and removed system in run experment wrapper --- Jenkinsfile | 5 +++-- ci/scripts/utils/ci_utils.sh | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 530ce7d16..556ed91a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -90,11 +90,12 @@ pipeline { } stage('Setup RUNTESTS') { + agent { label "${MACHINE}-emc" } steps { script { sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work - pullRequest.removeLabel("CI-${machine}-Building") + //pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } } @@ -130,7 +131,7 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") try { - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } catch (Exception e) { if (fileExists('${HOME}/RUNTESTS/ci.log')) { diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index f71221ccc..6b9312257 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -97,8 +97,6 @@ function create_experiment () { source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true - - "${HOME}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}" + "${HOME}/workflow/create_experiment.py" --yaml "${yaml_config}" } -- GitLab From 960e0fe2e42ca67ed2089b905d79193b2fede056 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 26 Jan 2024 22:53:32 +0000 Subject: [PATCH 316/447] updated path of HOMEgfs in run check scripts using gfs as default --- ci/scripts/run-check_ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 37761573d..f98f43446 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -21,7 +21,9 @@ pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this scr # │  └── ${pslot} # └── EXPDIR # └── ${pslot} -HOMEgfs="${TEST_DIR}" +# Two system build directories created at build time gfs, and gdas +# TODO: Make this configurable (for now all scripts run from gfs for CI at runtime) +HOMEgfs="${TEST_DIR}/gfs" RUNTESTS="${TEST_DIR}/RUNTESTS" # Source modules and setup logging -- GitLab From 0d09e5c15b5e7096ac1e61a4bc4d039e32f3b9fe Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 26 Jan 2024 22:57:05 +0000 Subject: [PATCH 317/447] use HOME to point to RUNDIRS for run-check --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 556ed91a9..20c314e7e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -131,7 +131,7 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") try { - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } catch (Exception e) { if (fileExists('${HOME}/RUNTESTS/ci.log')) { -- GitLab From 4eb89e24b46862783dac1e8c965340e11362cb25 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 27 Jan 2024 07:16:55 +0000 Subject: [PATCH 318/447] add label updates back move archive to post --- Jenkinsfile | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 20c314e7e..c85cc4952 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,6 +38,10 @@ pipeline { steps ( timeout(time: 1, unit: 'HOURS') ) { script { HOME = "${WORKSPACE}/TESTDIR" + pullRequest.addLabel("CI-${machine}-Building") + if (pullRequest.labels.contains("CI-${machine}-Ready")) { + pullRequest.removeLabel("CI-${machine}-Ready") + } } } } @@ -75,7 +79,7 @@ pipeline { } else if (system == "gefs") { // TODO: need to add gefs build arguments from a yaml file dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu", returnStatus: false) + sh( script: "./build_all.sh -gu -j 4", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } @@ -95,7 +99,7 @@ pipeline { script { sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work - //pullRequest.removeLabel("CI-${machine}-Building") + pullRequest.removeLabel("CI-${machine}-Building") pullRequest.addLabel("CI-${machine}-Running") } } @@ -134,15 +138,9 @@ pipeline { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } catch (Exception e) { - if (fileExists('${HOME}/RUNTESTS/ci.log')) { - def fileContent = readFile '${HOME}/RUNTESTS/ci.log' - fileContent.eachLine { line -> - archiveArtifacts artifacts: "${line}", fingerprint: true - } + pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") + error("Failed to run experiments ${Case} on ${machine}") } - } - pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") - error("Failed to run experiments ${Case} on ${machine}") } } } @@ -172,6 +170,12 @@ pipeline { pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + archiveArtifacts artifacts: "${line}", fingerprint: true + } + } } } } -- GitLab From 2e159cd77f92005e2bc08693730e7a4abc347b77 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 17:29:25 +0000 Subject: [PATCH 319/447] added throttle to build --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c85cc4952..64d69c184 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,13 +39,14 @@ pipeline { script { HOME = "${WORKSPACE}/TESTDIR" pullRequest.addLabel("CI-${machine}-Building") - if (pullRequest.labels.contains("CI-${machine}-Ready")) { + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") } } } } + properties([throttle(['Build'])]) stage('Build') { matrix { agent { label "${MACHINE}-emc" } -- GitLab From e1f054f343c92b8e5518b67bea69f5a09b5f0f34 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 17:33:53 +0000 Subject: [PATCH 320/447] remove prop from throttle --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 64d69c184..07e4f8148 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { } } - properties([throttle(['Build'])]) + throttle(['Build']) { stage('Build') { matrix { agent { label "${MACHINE}-emc" } @@ -92,7 +92,7 @@ pipeline { } } } - } + } } stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } -- GitLab From 03b551f72979bf806c9e0c3a8f02bc86a88eacf7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 17:41:46 +0000 Subject: [PATCH 321/447] moveed throttle inside options block of stage build --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07e4f8148..bc231800d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,10 +46,12 @@ pipeline { } } - throttle(['Build']) { stage('Build') { matrix { agent { label "${MACHINE}-emc" } + options { + throttle(['Building']) + } axes { axis { name "system" @@ -92,7 +94,7 @@ pipeline { } } } - } } + } stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } -- GitLab From 2329c4a877688d803d6123e373264c21c20f87bf Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 17:52:01 +0000 Subject: [PATCH 322/447] updated fixed label check code --- Jenkinsfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bc231800d..33f4733bf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -101,8 +101,9 @@ pipeline { steps { script { sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work - pullRequest.removeLabel("CI-${machine}-Building") + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { + pullRequest.removeLabel("CI-${machine}-Building") + } pullRequest.addLabel("CI-${machine}-Running") } } @@ -154,29 +155,29 @@ pipeline { } post { + always { + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) { + pullRequest.removeLabel("CI-${machine}-Running") + } + } success { script { - //if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) { - // pullRequest.removeLabel("CI-${machine}-Running") - //} - // TODO: contains mehthod does not work - pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") } - cleanWs() } failure { script { - pullRequest.removeLabel("CI-${machine}-Running") pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") if (fileExists('${HOME}/RUNTESTS/ci.log')) { def fileContent = readFile '${HOME}/RUNTESTS/ci.log' fileContent.eachLine { line -> - archiveArtifacts artifacts: "${line}", fingerprint: true + if( line.contains(".log")) { + archiveArtifacts artifacts: "${line}", fingerprint: true + } } } } -- GitLab From fc7a3a6abd901535472c9f45016e1d2228d01fb3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 17:54:53 +0000 Subject: [PATCH 323/447] ndded script block in post always --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33f4733bf..5eb9572a2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -156,8 +156,10 @@ pipeline { post { always { - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) { - pullRequest.removeLabel("CI-${machine}-Running") + script { + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) { + pullRequest.removeLabel("CI-${machine}-Running") + } } } success { -- GitLab From 5c4d4a6e089bddbe8f12217b618cc5c06d7854be Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 18:06:28 +0000 Subject: [PATCH 324/447] added confgired catigory to throttle --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5eb9572a2..1d8f273fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { matrix { agent { label "${MACHINE}-emc" } options { - throttle(['Building']) + throttle(['Matrix_build']) } axes { axis { -- GitLab From c0b386f70ae8f1137e5b38bf20c499b2f7f7383c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 18:29:38 +0000 Subject: [PATCH 325/447] took out throttle option and added limit j to 2 on both concurrent builds --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1d8f273fa..d9c5ace51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,9 +49,9 @@ pipeline { stage('Build') { matrix { agent { label "${MACHINE}-emc" } - options { - throttle(['Matrix_build']) - } + //options { + // throttle(['Matrix_build']) + //} axes { axis { name "system" @@ -75,14 +75,14 @@ pipeline { sh( script: "git submodule update --init --recursive", returnStatus: true) if (system == "gfs") { dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu -j 4", returnStatus: false) + sh( script: "./build_all.sh -gu -j 2", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { // TODO: need to add gefs build arguments from a yaml file dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu -j 4", returnStatus: false) + sh( script: "./build_all.sh -gu -j 2", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From e896a1c5a9238fd887e646dbfb3b204a2f957f36 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 29 Jan 2024 18:51:36 +0000 Subject: [PATCH 326/447] made remove all labels with machine in post alwasy a conventional loop --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9c5ace51..273ac500f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,8 +157,10 @@ pipeline { post { always { script { - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) { - pullRequest.removeLabel("CI-${machine}-Running") + for (label in pullRequest.labels) { + if (label.contains("${machine}")) { + pullRequest.removeLabel(label) + } } } } -- GitLab From ec432dafb6514abebd6dbeb2ba19ed6963e36168 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 29 Jan 2024 15:27:35 -0500 Subject: [PATCH 327/447] Update Jenkinsfile just removed one system and try just building gfs --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 273ac500f..024072ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,7 +55,7 @@ pipeline { axes { axis { name "system" - values "gfs", "gefs" + values "gfs" } } stages { @@ -75,7 +75,7 @@ pipeline { sh( script: "git submodule update --init --recursive", returnStatus: true) if (system == "gfs") { dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu -j 2", returnStatus: false) + sh( script: "./build_all.sh -gu -j 4", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From 189025956df91c0f83d7eb82cd76b8bdf6730353 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:12:32 +0000 Subject: [PATCH 328/447] added to matrix builds to make it sequential and added yaml reads --- Jenkinsfile | 122 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 024072ee8..b70695e64 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { } } - stage('Build') { + stage('Build GEFS') { matrix { agent { label "${MACHINE}-emc" } //options { @@ -55,6 +55,8 @@ pipeline { axes { axis { name "system" + //values "gfs", "gefs" + values "gefs" values "gfs" } } @@ -73,14 +75,122 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { sh( script: "git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" if (system == "gfs") { + def build_args = builds_file.find { it.system == system }.build_args + echo "build args: ${build_args}" + dir("${HOMEgfs}/sorc") { + sh( script: "./${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } + } else if (system == "gefs") { + def build_args = builds_file.find { it.system == system }.build_args + echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./build_all.sh -gu -j 4", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } + } + } + } + } + } + } + } + } + } + + + stage('Build GFS') { + matrix { + agent { label "${MACHINE}-emc" } + //options { + // throttle(['Matrix_build']) + //} + axes { + axis { + name "system" + //values "gfs", "gefs" + values "gfs" + } + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) { + checkout scm + env.MACHINE_ID = MACHINE + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } else { + sh( script: "git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + if (system == "gfs") { + def build_args = builds_file.find { it.system == system }.build_args + echo "build args: ${build_args}" + dir("${HOMEgfs}/sorc") { + sh( script: "./${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } + } + } + } + } + } + } + } + } + } + + + stage('Build GFS') { + matrix { + agent { label "${MACHINE}-emc" } + //options { + // throttle(['Matrix_build']) + //} + axes { + axis { + name "system" + //values "gfs", "gefs" + values "gfs" + } + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) { + checkout scm + env.MACHINE_ID = MACHINE + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } else { + sh( script: "git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + if (system == "gfs") { + def build_args = builds_file.find { it.system == system }.build_args + echo "build args: ${build_args}" + dir("${HOMEgfs}/sorc") { + sh( script: "./${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } } else if (system == "gefs") { - // TODO: need to add gefs build arguments from a yaml file + def build_args = builds_file.find { it.system == system }.build_args + echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./build_all.sh -gu -j 2", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) @@ -96,6 +206,7 @@ pipeline { } } + stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } steps { @@ -123,11 +234,10 @@ pipeline { stage('Create Experiment') { steps { script { + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml" + system = yaml_case.experiment.system + def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - def HOMEgfs = "${HOME}/gfs" - env.HOME = HOMEgfs - sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" ) - sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" ) sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } -- GitLab From f3be7f36df6b00cdba38f269fe23f72f8551766f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:18:13 +0000 Subject: [PATCH 329/447] had extra values line in first matrix build --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b70695e64..ae4e682e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,7 +57,6 @@ pipeline { name "system" //values "gfs", "gefs" values "gefs" - values "gfs" } } stages { -- GitLab From 8a263936b3bfb7853ff23cf299978e2efa2561b1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:34:28 +0000 Subject: [PATCH 330/447] had an extra pasted GFS build --- Jenkinsfile | 57 ----------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae4e682e7..9e1b75a6d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -149,63 +149,6 @@ pipeline { } } - - stage('Build GFS') { - matrix { - agent { label "${MACHINE}-emc" } - //options { - // throttle(['Matrix_build']) - //} - axes { - axis { - name "system" - //values "gfs", "gefs" - values "gfs" - } - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { - checkout scm - env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - } else { - sh( script: "git submodule update --init --recursive", returnStatus: true) - def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - if (system == "gfs") { - def build_args = builds_file.find { it.system == system }.build_args - echo "build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "./${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } else if (system == "gefs") { - def build_args = builds_file.find { it.system == system }.build_args - echo "build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu -j 2", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } - } - } - } - } - } - } - } - } - - stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } steps { -- GitLab From cbf32bc48979786a35a92f92b1952cdbb176b669 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:41:18 +0000 Subject: [PATCH 331/447] added build yaml file in cases yamls --- ci/cases/yamls/build.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 ci/cases/yamls/build.yaml diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml new file mode 100644 index 000000000..20c80fe29 --- /dev/null +++ b/ci/cases/yamls/build.yaml @@ -0,0 +1,3 @@ +builds: +- gefs: './build_all.sh -gu' +- gfs: './build_all.sh -gu' -- GitLab From bed6fa8bf7047fa8b96ab813730154c24c67931c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:51:14 +0000 Subject: [PATCH 332/447] variable key had to be imutable --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9e1b75a6d..7cd2747de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,7 +76,8 @@ pipeline { sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" if (system == "gfs") { - def build_args = builds_file.find { it.system == system }.build_args + def system_key = string( system ) + def build_args = builds_file.find { it.system_key == system_key }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./${build_args}", returnStatus: false) @@ -84,7 +85,8 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { - def build_args = builds_file.find { it.system == system }.build_args + def system_key = string( system ) + def build_args = builds_file.find { it.system_key == system_key }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./build_all.sh -gu -j 4", returnStatus: false) -- GitLab From e4426fcb4587b6c3ac1bb5773c223f031a5cedb1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 19:55:00 +0000 Subject: [PATCH 333/447] testing imuable key --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7cd2747de..f64b6d4fa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,8 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { - def system_key = string( system ) + // def system_key = string( system ) + def system_key = "gefs" def build_args = builds_file.find { it.system_key == system_key }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { -- GitLab From d8bbea93972d439554e5a3eee7ff31574162dc47 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 20:11:43 +0000 Subject: [PATCH 334/447] testing yaml output --- Jenkinsfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f64b6d4fa..9408cbf9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -67,17 +67,17 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { - checkout scm + //checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - sh( script: "git submodule update --init --recursive", returnStatus: true) + //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + def build_args = builds_file['builds'] + echo "build args: ${build_args}" if (system == "gfs") { - def system_key = string( system ) - def build_args = builds_file.find { it.system_key == system_key }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./${build_args}", returnStatus: false) @@ -85,9 +85,6 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } else if (system == "gefs") { - // def system_key = string( system ) - def system_key = "gefs" - def build_args = builds_file.find { it.system_key == system_key }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./build_all.sh -gu -j 4", returnStatus: false) -- GitLab From 4a03f8760f0f5be440aae8765c490795301f7f1b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 20:26:39 +0000 Subject: [PATCH 335/447] traverse build args --- Jenkinsfile | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9408cbf9d..1fae34f10 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,21 +76,15 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args = builds_file['builds'] + build_args.value.each{ build -> + if (build.key == system) { + build_args = build.value + } echo "build args: ${build_args}" - if (system == "gfs") { - echo "build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "./${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } else if (system == "gefs") { - echo "build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "./build_all.sh -gu -j 4", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } + dir("${HOMEgfs}/sorc") { + sh( script: "${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } } -- GitLab From 51ab85f6bc78de24b9ce80adaa7db3cd6a240a5f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 20:31:54 +0000 Subject: [PATCH 336/447] missed } on iter --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1fae34f10..ff22e226a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,12 +77,13 @@ pipeline { def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args = builds_file['builds'] build_args.value.each{ build -> - if (build.key == system) { - build_args = build.value + if (build.key == system) { + build_args = build.value + } } echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: false) + sh( script: "", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From f2a018a3cf59c56494ca3dfa171503fa27f5643c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 20:42:29 +0000 Subject: [PATCH 337/447] use get in yaml --- Jenkinsfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ff22e226a..a8b43f0b2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,12 +75,8 @@ pipeline { } else { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - def build_args = builds_file['builds'] - build_args.value.each{ build -> - if (build.key == system) { - build_args = build.value - } - } + def build_args_list = builds_file['builds'] + def build_args = build_args_list.get(system) echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) -- GitLab From 59f7aeeb1a7ab19204312c2a2d93c090c371c510 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 20:59:39 +0000 Subject: [PATCH 338/447] try it key --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8b43f0b2..81075c161 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,7 +76,7 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] - def build_args = build_args_list.get(system) + def build_args = build_args_list.find { it.key == system }.value echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) -- GitLab From 2910b53dcaee864677be4f4e59c081bd7ecf55e3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:15:49 +0000 Subject: [PATCH 339/447] just system --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 81075c161..4909c2dd2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,7 +76,8 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] - def build_args = build_args_list.find { it.key == system }.value + echo "build args list: ${build_args_list}" + def build_args = build_args_list[system] echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) -- GitLab From cbc9e2282cc3ce66a52f99d82d61e6fd9ec7b602 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:25:32 +0000 Subject: [PATCH 340/447] to string --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4909c2dd2..a1de84130 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ pipeline { def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] echo "build args list: ${build_args_list}" - def build_args = build_args_list[system] + def build_args = build_args_list[system].toString() echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) -- GitLab From b3738650de5fb21017ad63c158a73f560d97572e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:32:55 +0000 Subject: [PATCH 341/447] to string --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a1de84130..e6ee571e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ pipeline { def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] echo "build args list: ${build_args_list}" - def build_args = build_args_list[system].toString() + def build_args = build_args_list[system].join() echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) -- GitLab From a0fd397894ba2e019980f701299a51af17a6ed66 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:39:57 +0000 Subject: [PATCH 342/447] still trying --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e6ee571e6..784ba7608 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,8 +77,10 @@ pipeline { def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] echo "build args list: ${build_args_list}" - def build_args = build_args_list[system].join() + def build_args = build_args_list[system] echo "build args: ${build_args}" + def test = build_args.join(" ") + echo "test: ${test}" dir("${HOMEgfs}/sorc") { sh( script: "", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) @@ -115,16 +117,16 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { - checkout scm + //checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - sh( script: "git submodule update --init --recursive", returnStatus: true) + //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" if (system == "gfs") { - def build_args = builds_file.find { it.system == system }.build_args + //def build_args = builds_file.find { it.system == system }.build_args echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "./${build_args}", returnStatus: false) -- GitLab From 09d6eb82d288298e7844cc71730d9c5910a33270 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:46:32 +0000 Subject: [PATCH 343/447] still foo --- Jenkinsfile | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 784ba7608..fde9072bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -96,53 +96,6 @@ pipeline { } - stage('Build GFS') { - matrix { - agent { label "${MACHINE}-emc" } - //options { - // throttle(['Matrix_build']) - //} - axes { - axis { - name "system" - //values "gfs", "gefs" - values "gfs" - } - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { - //checkout scm - env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - } else { - //sh( script: "git submodule update --init --recursive", returnStatus: true) - def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - if (system == "gfs") { - //def build_args = builds_file.find { it.system == system }.build_args - echo "build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "./${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } - } - } - } - } - } - } - } - } - stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } steps { -- GitLab From 01e09baecf493a4564c2a01c0645ba1451b19b24 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:53:46 +0000 Subject: [PATCH 344/447] finally --- Jenkinsfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fde9072bb..1b9fcd728 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,13 +76,10 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] - echo "build args list: ${build_args_list}" - def build_args = build_args_list[system] + def build_args = build_args_list[system].join(" ") echo "build args: ${build_args}" - def test = build_args.join(" ") - echo "test: ${test}" dir("${HOMEgfs}/sorc") { - sh( script: "", returnStatus: false) + sh( script: "${build_args}", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } -- GitLab From bb7ff253d3d54d46dcd8fcf2fa3a79fb447f930f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 21:56:46 +0000 Subject: [PATCH 345/447] finally then trim --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1b9fcd728..76aad3bbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,7 +76,7 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] - def build_args = build_args_list[system].join(" ") + def build_args = build_args_list[system].join(" ").trim() echo "build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "${build_args}", returnStatus: false) -- GitLab From 01567473201624bf473f35e26052f98c1102d16a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 22:03:07 +0000 Subject: [PATCH 346/447] finally then trim and nulls --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 76aad3bbe..b3a4b6f75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,8 +76,8 @@ pipeline { //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] - def build_args = build_args_list[system].join(" ").trim() - echo "build args: ${build_args}" + def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") + echo "trim build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "${build_args}", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) -- GitLab From e9a6b70dbebd0796acd2b0403a8e5e955f0814d4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Tue, 30 Jan 2024 22:06:11 +0000 Subject: [PATCH 347/447] got args to work --- Jenkinsfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index b3a4b6f75..2123f8c90 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,6 +46,52 @@ pipeline { } } + stage('Build GFS') { + matrix { + agent { label "${MACHINE}-emc" } + //options { + // throttle(['Matrix_build']) + //} + axes { + axis { + name "system" + //values "gfs", "gefs" + values "gfs" + } + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) { + //checkout scm + env.MACHINE_ID = MACHINE + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } else { + //sh( script: "git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + def build_args_list = builds_file['builds'] + def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") + echo "trim build args: ${build_args}" + dir("${HOMEgfs}/sorc") { + sh( script: "${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } + } + } + } + } + } + } + } + } + stage('Build GEFS') { matrix { agent { label "${MACHINE}-emc" } @@ -93,6 +139,7 @@ pipeline { } + stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } steps { -- GitLab From c4dcbe68b89244987f41fdc5a5d276b996bff2c3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 01:01:56 +0000 Subject: [PATCH 348/447] put clone back in --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2123f8c90..2835c719c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,13 +113,13 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { - //checkout scm + checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - //sh( script: "git submodule update --init --recursive", returnStatus: true) + sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From ec6ba94c290d4c58b91a8e510dbb42da228812b2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 15:51:59 +0000 Subject: [PATCH 349/447] added back throttled build and removed second build block --- Jenkinsfile | 59 +++++------------------------------------------------ 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2835c719c..85f7373ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,63 +46,17 @@ pipeline { } } - stage('Build GFS') { + stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - //options { - // throttle(['Matrix_build']) - //} - axes { - axis { - name "system" - //values "gfs", "gefs" - values "gfs" - } - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { - //checkout scm - env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - } else { - //sh( script: "git submodule update --init --recursive", returnStatus: true) - def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - def build_args_list = builds_file['builds'] - def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") - echo "trim build args: ${build_args}" - dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } - } - } - } - } + options { + throttle(['global_matrix_build']) } - } - } - - stage('Build GEFS') { - matrix { - agent { label "${MACHINE}-emc" } - //options { - // throttle(['Matrix_build']) - //} axes { axis { name "system" - //values "gfs", "gefs" - values "gefs" + values "gfs", "gefs" + values "gfs" } } stages { @@ -123,7 +77,6 @@ pipeline { def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") - echo "trim build args: ${build_args}" dir("${HOMEgfs}/sorc") { sh( script: "${build_args}", returnStatus: false) sh( script: "./link_workflow.sh", returnStatus: false) @@ -138,8 +91,6 @@ pipeline { } } - - stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } steps { -- GitLab From 85522b7ee985d4139ba9583068662cf9ac2486b2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 16:03:42 +0000 Subject: [PATCH 350/447] removed extra leftover values line in build matrix --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 85f7373ce..bcecca037 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,7 +56,6 @@ pipeline { axis { name "system" values "gfs", "gefs" - values "gfs" } } stages { -- GitLab From 1c77e99b0f0bc0a320dd950ffd72862963a67afa Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 18:32:24 +0000 Subject: [PATCH 351/447] removed non yaml lines from cases so snakeyaml could read it --- Jenkinsfile | 3 ++- ci/scripts/utils/ci_utils.sh | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bcecca037..b861369c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,7 +117,8 @@ pipeline { stage('Create Experiment') { steps { script { - def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml" + sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 6b9312257..340288a38 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -97,6 +97,8 @@ function create_experiment () { source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - "${HOME}/workflow/create_experiment.py" --yaml "${yaml_config}" + # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true + + "${HOMEgfs}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}" } -- GitLab From 773164aa12741e94bf91dc5c1eb8454ad187865f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 19:53:15 +0000 Subject: [PATCH 352/447] added override in create experment and removed stubed scripts --- ci/scripts/run-check_ci_stub.sh | 11 ----------- ci/scripts/utils/ci_utils.sh | 2 +- sorc/build_all_stub.sh | 5 ----- 3 files changed, 1 insertion(+), 17 deletions(-) delete mode 100755 ci/scripts/run-check_ci_stub.sh delete mode 100755 sorc/build_all_stub.sh diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh deleted file mode 100755 index 23d3da837..000000000 --- a/ci/scripts/run-check_ci_stub.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -eu - -HOMEgfs=$1 -pslot=$2 - -echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log" -if [[ ${pslot} =~ "C48_S2SW" ]]; then - exit 1 -fi diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 340288a38..7422d3457 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -99,6 +99,6 @@ function create_experiment () { # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true - "${HOMEgfs}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}" + "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" } diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh deleted file mode 100755 index 6221b6bb5..000000000 --- a/sorc/build_all_stub.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -eu - -echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)" -- GitLab From 9747d7affec401d554260de48ac99edbd0cf3c19 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Wed, 31 Jan 2024 20:33:10 +0000 Subject: [PATCH 353/447] moved scm after semephore test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b861369c2..0bb9f747c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,12 +66,12 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { - checkout scm env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { + checkout scm sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] -- GitLab From 6d439b66068b539d4d9d2e4f511a4393dbf9ddc0 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 31 Jan 2024 18:01:25 -0500 Subject: [PATCH 354/447] Update Jenkinsfile try a build without a manual git submodule recursive and see if scm can do it correctly --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0bb9f747c..d829b1e62 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,7 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { checkout scm - sh( script: "git submodule update --init --recursive", returnStatus: true) + //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From a7ff014d423ef45767f0d37edad9720085435440 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 31 Jan 2024 18:04:40 -0500 Subject: [PATCH 355/447] Update Jenkinsfile no for separate PR test on Orion --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d829b1e62..f9b7ee322 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ def MACHINE = 'none' def machine = 'none' def HOME = 'none' - +// NO OP for Orion seperate PR pipeline { agent { label 'built-in' } -- GitLab From c2e0f82c8b2191edb772c6444ef16b9bbe0a0a4e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Wed, 31 Jan 2024 21:09:47 -0500 Subject: [PATCH 356/447] Update Jenkinsfile put back in submodle upate --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f9b7ee322..09a2c128f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,7 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { checkout scm - //sh( script: "git submodule update --init --recursive", returnStatus: true) + sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From da9f7cd11a52a3e2b382802307a60e50a90eed67 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:05:51 +0000 Subject: [PATCH 357/447] added pre checkout in TESTDIR added source of gwsetup and changed first dir to ws --- Jenkinsfile | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 09a2c128f..5fc45fa06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,15 @@ def MACHINE = 'none' def machine = 'none' def HOME = 'none' -// NO OP for Orion seperate PR + +checkout([ + $class: 'GitSCM', + branches: scm.branches, + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], + userRemoteConfigs: scm.userRemoteConfigs +]) + pipeline { agent { label 'built-in' } @@ -37,7 +45,10 @@ pipeline { agent { label "${MACHINE}-emc" } steps ( timeout(time: 1, unit: 'HOURS') ) { script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" + sh( script: "mkdir -p ${HOME}", returnStatus: true) + checkout pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -63,16 +74,16 @@ pipeline { steps { script { def HOMEgfs = "${HOME}/${system}" - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { + ws(HOMEgfs) { env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { + sh( script: "source ${HOME/workflow/gw_setup.sh}", returnStatus: true) checkout scm - sh( script: "git submodule update --init --recursive", returnStatus: true) + //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From 3662f07d878fffc03d227ee9a9c245d1953b0d49 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:12:06 +0000 Subject: [PATCH 358/447] added git to hera gw setup module load --- modulefiles/module_gwsetup.hera.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index c86cac7b0..ab7223372 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -3,6 +3,7 @@ Load environment to run GFS workflow setup scripts on Hera ]]) load(pathJoin("rocoto")) +load(pathJoin("git","2.40.0")) prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -- GitLab From 5c2eb59b1e691bf05cc4dfff3ad4717fb6c50a17 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:20:02 +0000 Subject: [PATCH 359/447] made shallow checkout a function call --- Jenkinsfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5fc45fa06..dc82e2b4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,13 +2,15 @@ def MACHINE = 'none' def machine = 'none' def HOME = 'none' -checkout([ - $class: 'GitSCM', - branches: scm.branches, - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], - userRemoteConfigs: scm.userRemoteConfigs -]) +def checkout_shallow () { + checkout([ + $class: 'GitSCM', + branches: scm.branches, + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], + userRemoteConfigs: scm.userRemoteConfigs + ]) +} pipeline { agent { label 'built-in' } @@ -48,7 +50,7 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" sh( script: "mkdir -p ${HOME}", returnStatus: true) - checkout + checkout_shallow() pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") -- GitLab From be7b72e5747efedd0aa06d0b9db8b277e2c76188 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:23:09 +0000 Subject: [PATCH 360/447] misplaced bracket in source of gw_setup --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc82e2b4f..5eaf25970 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,7 +83,7 @@ pipeline { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - sh( script: "source ${HOME/workflow/gw_setup.sh}", returnStatus: true) + sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true) checkout scm //sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" -- GitLab From aeaa5cfba670bdc5f5739cd7427834c54b089b8f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:34:35 +0000 Subject: [PATCH 361/447] cd into TESTDIR for shallow clone --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5eaf25970..856245acc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,7 +50,9 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" sh( script: "mkdir -p ${HOME}", returnStatus: true) - checkout_shallow() + dir("${HOME}") { + checkout_shallow() + } pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") -- GitLab From 8fa786824e021709bcfe4e9e93635e555426e6c6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:42:47 +0000 Subject: [PATCH 362/447] moved git load on gwset hera because it is in the stack spac --- modulefiles/module_gwsetup.hera.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index ab7223372..244a2419e 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -3,7 +3,6 @@ Load environment to run GFS workflow setup scripts on Hera ]]) load(pathJoin("rocoto")) -load(pathJoin("git","2.40.0")) prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") @@ -15,5 +14,6 @@ load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") +load(pathJoin("git","2.40.0")) whatis("Description: GFS run setup environment") -- GitLab From 76f13ed3b9b058c00a7e96bd8fcfd08fe9c1fcab Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 15:57:20 +0000 Subject: [PATCH 363/447] use shallow checkout and git submoudle recussive update directly --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 856245acc..e7b1f5317 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,8 +86,8 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true) - checkout scm - //sh( script: "git submodule update --init --recursive", returnStatus: true) + checkout_shallow() + sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From 2685a85cf7f0428c1b9750543642aaa4e53c43c7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 17:49:03 +0000 Subject: [PATCH 364/447] configured scm shallow and using git modules recursive from host --- Jenkinsfile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e7b1f5317..324d166e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,16 +2,6 @@ def MACHINE = 'none' def machine = 'none' def HOME = 'none' -def checkout_shallow () { - checkout([ - $class: 'GitSCM', - branches: scm.branches, - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], - userRemoteConfigs: scm.userRemoteConfigs - ]) -} - pipeline { agent { label 'built-in' } @@ -86,7 +76,7 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true) - checkout_shallow() + checkout scm sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] -- GitLab From 0944fd418dea5e94b758bb3cbe8ea7d5f6b6feeb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 17:53:13 +0000 Subject: [PATCH 365/447] still had shallow call and removed it --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 324d166e7..1ee23a771 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ pipeline { HOME = "${WORKSPACE}/TESTDIR" sh( script: "mkdir -p ${HOME}", returnStatus: true) dir("${HOME}") { - checkout_shallow() + checkout scm } pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { -- GitLab From a1ba03b4d037caddaa246e003334be8ea041b6c2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 18:18:48 +0000 Subject: [PATCH 366/447] put dir back at attempt to use ws --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ee23a771..cd7d890a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,7 +69,7 @@ pipeline { script { def HOMEgfs = "${HOME}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - ws(HOMEgfs) { + dir(HOMEgfs) { env.MACHINE_ID = MACHINE if (fileExists("sorc/BUILT_semaphor")) { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() -- GitLab From de514d90c2f7d2d59dbb663f3f6aa0855c87fc3f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Thu, 1 Feb 2024 19:36:18 +0000 Subject: [PATCH 367/447] take out pre checkout and just use submodule update directly --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cd7d890a6..4ef3cf167 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,9 +40,6 @@ pipeline { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" sh( script: "mkdir -p ${HOME}", returnStatus: true) - dir("${HOME}") { - checkout scm - } pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -75,7 +72,6 @@ pipeline { sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true) checkout scm sh( script: "git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" -- GitLab From a80666e0815b21e2bc31d94d34933ea2b955f598 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 16:23:37 +0000 Subject: [PATCH 368/447] stub build all to just gdas --- Jenkinsfile | 2 +- ci/cases/yamls/build.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4ef3cf167..fdb23aadf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,7 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { checkout scm - sh( script: "git submodule update --init --recursive", returnStatus: true) + sh( script: "source workflow/gw_setup.sh;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 20c80fe29..663409ee2 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,5 @@ builds: -- gefs: './build_all.sh -gu' -- gfs: './build_all.sh -gu' +#- gefs: './build_all.sh -gu' +#- gfs: './build_all.sh -gu' +- gefs: './build_gdas.sh -j 4' +- gfs: './build_gdas.sh -j 8' -- GitLab From 2e43230d29ca16397c3d80624129f1ba206f38df Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 16:24:49 +0000 Subject: [PATCH 369/447] added git which and version for debugging --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fdb23aadf..435431226 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,7 @@ pipeline { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { checkout scm - sh( script: "source workflow/gw_setup.sh;git submodule update --init --recursive", returnStatus: true) + sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") -- GitLab From 5071db10aee01ecb00aa639382cee76e77b3cbbc Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 16:43:49 +0000 Subject: [PATCH 370/447] put buile lines back in build yaml --- ci/cases/yamls/build.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 663409ee2..20c80fe29 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,5 +1,3 @@ builds: -#- gefs: './build_all.sh -gu' -#- gfs: './build_all.sh -gu' -- gefs: './build_gdas.sh -j 4' -- gfs: './build_gdas.sh -j 8' +- gefs: './build_all.sh -gu' +- gfs: './build_all.sh -gu' -- GitLab From 948992c3e6f50414aa8717649e226c423b9b6184 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 17:24:46 +0000 Subject: [PATCH 371/447] added a second Jenkins file to test dynamic parallel build stages --- Jenkinsfile2 | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 Jenkinsfile2 diff --git a/Jenkinsfile2 b/Jenkinsfile2 new file mode 100644 index 000000000..435431226 --- /dev/null +++ b/Jenkinsfile2 @@ -0,0 +1,186 @@ +def MACHINE = 'none' +def machine = 'none' +def HOME = 'none' + +pipeline { + agent { label 'built-in' } + + options { + disableConcurrentBuilds(abortPrevious: true) + skipDefaultCheckout(true) + buildDiscarder(logRotator(numToKeepStr: '3')) + } + + stages { + + stage('Get Machine') { + agent { label 'built-in' } + steps { + script { + MACHINE = 'none' + for (label in pullRequest.labels) { + echo "Label: ${label}" + if ((label.matches("CI-Hera-Ready"))) { + MACHINE = 'hera' + } else if ((label.matches("CI-Orion-Ready"))) { + MACHINE = 'orion' + } else if ((label.matches("CI-Hercules-Ready"))) { + MACHINE = 'hercules' + } + } + machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + } + } + } + + stage('Get Common Workspace') { + agent { label "${MACHINE}-emc" } + steps ( timeout(time: 1, unit: 'HOURS') ) { + script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + HOME = "${WORKSPACE}/TESTDIR" + sh( script: "mkdir -p ${HOME}", returnStatus: true) + pullRequest.addLabel("CI-${machine}-Building") + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { + pullRequest.removeLabel("CI-${machine}-Ready") + } + } + } + } + + stage('Build System') { + matrix { + agent { label "${MACHINE}-emc" } + options { + throttle(['global_matrix_build']) + } + axes { + axis { + name "system" + values "gfs", "gefs" + } + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + dir(HOMEgfs) { + env.MACHINE_ID = MACHINE + if (fileExists("sorc/BUILT_semaphor")) { + sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + } else { + checkout scm + sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + def build_args_list = builds_file['builds'] + def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") + dir("${HOMEgfs}/sorc") { + sh( script: "${build_args}", returnStatus: false) + sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } + } + } + } + } + } + } + } + } + + stage('Setup RUNTESTS') { + agent { label "${MACHINE}-emc" } + steps { + script { + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { + pullRequest.removeLabel("CI-${machine}-Building") + } + pullRequest.addLabel("CI-${machine}-Running") + } + } + + } + + stage('Run Tests') { + matrix { + agent { label "${MACHINE}-emc" } + axes { + axis { + name "Case" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + } + } + stages { + stage('Create Experiment') { + steps { + script { + sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" + system = yaml_case.experiment.system + def HOMEgfs = "${HOME}/${system}" + env.RUNTESTS = "${HOME}/RUNTESTS" + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + } + } + } + stage('Run Experiments') { + steps { + script { + def HOMEgfs = "${HOME}/gfs" + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + try { + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + } catch (Exception e) { + pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") + error("Failed to run experiments ${Case} on ${machine}") + } + } + } + } + } + } + } + + } + + post { + always { + script { + for (label in pullRequest.labels) { + if (label.contains("${machine}")) { + pullRequest.removeLabel(label) + } + } + } + } + success { + script { + pullRequest.addLabel("CI-${machine}-Passed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") + } + } + failure { + script { + pullRequest.addLabel("CI-${machine}-Failed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + if( line.contains(".log")) { + archiveArtifacts artifacts: "${line}", fingerprint: true + } + } + } + } + } + } + +} -- GitLab From e7790ee91ce18e1ce94258c2a60a7bd251fecc7a Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 17:31:29 +0000 Subject: [PATCH 372/447] removed Jenkinsfile2 to put it into its own repo --- Jenkinsfile2 | 186 --------------------------------------------------- 1 file changed, 186 deletions(-) delete mode 100644 Jenkinsfile2 diff --git a/Jenkinsfile2 b/Jenkinsfile2 deleted file mode 100644 index 435431226..000000000 --- a/Jenkinsfile2 +++ /dev/null @@ -1,186 +0,0 @@ -def MACHINE = 'none' -def machine = 'none' -def HOME = 'none' - -pipeline { - agent { label 'built-in' } - - options { - disableConcurrentBuilds(abortPrevious: true) - skipDefaultCheckout(true) - buildDiscarder(logRotator(numToKeepStr: '3')) - } - - stages { - - stage('Get Machine') { - agent { label 'built-in' } - steps { - script { - MACHINE = 'none' - for (label in pullRequest.labels) { - echo "Label: ${label}" - if ((label.matches("CI-Hera-Ready"))) { - MACHINE = 'hera' - } else if ((label.matches("CI-Orion-Ready"))) { - MACHINE = 'orion' - } else if ((label.matches("CI-Hercules-Ready"))) { - MACHINE = 'hercules' - } - } - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) - } - } - } - - stage('Get Common Workspace') { - agent { label "${MACHINE}-emc" } - steps ( timeout(time: 1, unit: 'HOURS') ) { - script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOME = "${WORKSPACE}/TESTDIR" - sh( script: "mkdir -p ${HOME}", returnStatus: true) - pullRequest.addLabel("CI-${machine}-Building") - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { - pullRequest.removeLabel("CI-${machine}-Ready") - } - } - } - } - - stage('Build System') { - matrix { - agent { label "${MACHINE}-emc" } - options { - throttle(['global_matrix_build']) - } - axes { - axis { - name "system" - values "gfs", "gefs" - } - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { - env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") - } else { - checkout scm - sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) - def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - def build_args_list = builds_file['builds'] - def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") - dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - } - } - } - } - } - } - } - } - } - - stage('Setup RUNTESTS') { - agent { label "${MACHINE}-emc" } - steps { - script { - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { - pullRequest.removeLabel("CI-${machine}-Building") - } - pullRequest.addLabel("CI-${machine}-Running") - } - } - - } - - stage('Run Tests') { - matrix { - agent { label "${MACHINE}-emc" } - axes { - axis { - name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - } - } - stages { - stage('Create Experiment') { - steps { - script { - sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) - def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" - system = yaml_case.experiment.system - def HOMEgfs = "${HOME}/${system}" - env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - } - } - } - stage('Run Experiments') { - steps { - script { - def HOMEgfs = "${HOME}/gfs" - pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - try { - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") - } catch (Exception e) { - pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") - error("Failed to run experiments ${Case} on ${machine}") - } - } - } - } - } - } - } - - } - - post { - always { - script { - for (label in pullRequest.labels) { - if (label.contains("${machine}")) { - pullRequest.removeLabel(label) - } - } - } - } - success { - script { - pullRequest.addLabel("CI-${machine}-Passed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") - } - } - failure { - script { - pullRequest.addLabel("CI-${machine}-Failed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") - if (fileExists('${HOME}/RUNTESTS/ci.log')) { - def fileContent = readFile '${HOME}/RUNTESTS/ci.log' - fileContent.eachLine { line -> - if( line.contains(".log")) { - archiveArtifacts artifacts: "${line}", fingerprint: true - } - } - } - } - } - } - -} -- GitLab From 29f61f16613f468efc3063fe211b9f9af60a459f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 18:55:07 +0000 Subject: [PATCH 373/447] adding deleteDir to get scm to checkout again --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 435431226..5a33c0e51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,10 +68,11 @@ pipeline { sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { env.MACHINE_ID = MACHINE - if (fileExists("sorc/BUILT_semaphor")) { - sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim() + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { + sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { + deleteDir() checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" -- GitLab From 74ff69980a275b5f5a0f54415c07a70dc133d80b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 18:57:01 +0000 Subject: [PATCH 374/447] adding build stubs --- ci/cases/yamls/build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 20c80fe29..6d395d80b 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,5 @@ builds: -- gefs: './build_all.sh -gu' -- gfs: './build_all.sh -gu' +#- gefs: './build_all.sh -gu' +#- gfs: './build_all.sh -gu' + - gefs: 'echo "Skip gefs build' + - gfs: 'echo "Skip gfs build' -- GitLab From 2978c6caf630b6a2f4ca04de2ed7c3439ffaaa08 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 19:05:34 +0000 Subject: [PATCH 375/447] adding build stubs again --- ci/cases/yamls/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 6d395d80b..357f92f91 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,5 +1,5 @@ builds: #- gefs: './build_all.sh -gu' #- gfs: './build_all.sh -gu' - - gefs: 'echo "Skip gefs build' - - gfs: 'echo "Skip gfs build' + - gefs: 'echo "Skip gefs build"' + - gfs: 'echo "Skip gfs build"' -- GitLab From af48ce8f251d1caf97c235cd808653477498f2ae Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 19:38:24 +0000 Subject: [PATCH 376/447] forgot to add build.yaml updates --- Jenkinsfile | 6 +++--- ci/cases/yamls/build.yaml | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5a33c0e51..be3cbe00b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,9 +51,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - options { - throttle(['global_matrix_build']) - } + //options { + // throttle(['global_matrix_build']) + //} axes { axis { name "system" diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 357f92f91..56fce131d 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,5 +1,3 @@ builds: -#- gefs: './build_all.sh -gu' -#- gfs: './build_all.sh -gu' - - gefs: 'echo "Skip gefs build"' - - gfs: 'echo "Skip gfs build"' + - gefs: './build_all.sh -gu' + - gfs: './build_all.sh -gu' \ No newline at end of file -- GitLab From 348c758c58cfd0fb2911eec2d551144bd58e0b86 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Fri, 2 Feb 2024 21:29:56 +0000 Subject: [PATCH 377/447] put throttle back and updated build args for gefs --- Jenkinsfile | 6 +++--- ci/cases/yamls/build.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be3cbe00b..5a33c0e51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,9 +51,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - //options { - // throttle(['global_matrix_build']) - //} + options { + throttle(['global_matrix_build']) + } axes { axis { name "system" diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 56fce131d..5398fa188 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - - gefs: './build_all.sh -gu' + - gefs: './build_all.sh' - gfs: './build_all.sh -gu' \ No newline at end of file -- GitLab From a0386edaf46795f2e7234c60d1e689bb791e888d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 3 Feb 2024 03:18:01 +0000 Subject: [PATCH 378/447] use rsync to get clones before submodules --- Jenkinsfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5a33c0e51..01d5be9ba 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,13 @@ pipeline { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" - sh( script: "mkdir -p ${HOME}", returnStatus: true) + sh( script: "mkdir -p ${HOME}/repo_pr", returnStatus: true) + dir("${HOME}/repo_pr") { + checkout scm + } + // TODO get system build dirs from ${HOME}/repo_pr/ci/cases/yamls/build.yaml + sh( script: "rsync -p ${HOME}/repo_pr ${HOME}/gfs", returnStatus: true) + sh( script: "rsync -p ${HOME}/repo_pr ${HOME}/gefs", returnStatus: true) pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -72,8 +78,6 @@ pipeline { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - deleteDir() - checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] -- GitLab From 417688c85474c448390395b119ecc7fd1d4eaaac Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 3 Feb 2024 03:36:09 +0000 Subject: [PATCH 379/447] update rsync line --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 01d5be9ba..0f41524cb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,8 +44,8 @@ pipeline { checkout scm } // TODO get system build dirs from ${HOME}/repo_pr/ci/cases/yamls/build.yaml - sh( script: "rsync -p ${HOME}/repo_pr ${HOME}/gfs", returnStatus: true) - sh( script: "rsync -p ${HOME}/repo_pr ${HOME}/gefs", returnStatus: true) + sh( script: "rsync -a ${HOME}/repo_pr/ ${HOME}/gfs", returnStatus: true) + sh( script: "rsync -a ${HOME}/repo_pr/ ${HOME}/gefs", returnStatus: true) pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -71,7 +71,7 @@ pipeline { steps { script { def HOMEgfs = "${HOME}/${system}" - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + // sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { -- GitLab From 240a2622b36f8b0ed81411150b0b2fd346f9d624 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sat, 3 Feb 2024 12:43:54 +0000 Subject: [PATCH 380/447] revert to scm and comment out disable current builds --- Jenkinsfile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0f41524cb..e33dbdc4d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ pipeline { agent { label 'built-in' } options { - disableConcurrentBuilds(abortPrevious: true) + //t disableConcurrentBuilds(abortPrevious: false) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '3')) } @@ -39,13 +39,7 @@ pipeline { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" - sh( script: "mkdir -p ${HOME}/repo_pr", returnStatus: true) - dir("${HOME}/repo_pr") { - checkout scm - } - // TODO get system build dirs from ${HOME}/repo_pr/ci/cases/yamls/build.yaml - sh( script: "rsync -a ${HOME}/repo_pr/ ${HOME}/gfs", returnStatus: true) - sh( script: "rsync -a ${HOME}/repo_pr/ ${HOME}/gefs", returnStatus: true) + sh( script: "mkdir -p ${HOME}", returnStatus: true) pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -71,13 +65,15 @@ pipeline { steps { script { def HOMEgfs = "${HOME}/${system}" - // sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) dir(HOMEgfs) { env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { + deleteDir() + checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] -- GitLab From 379958448f7a24e0e4d149ee5c93f2723a0fc96a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Sun, 4 Feb 2024 14:45:05 -0500 Subject: [PATCH 381/447] Update Jenkinsfile edit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e33dbdc4d..e4b6baded 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ pipeline { agent { label 'built-in' } options { - //t disableConcurrentBuilds(abortPrevious: false) + // disableConcurrentBuilds(abortPrevious: false) skipDefaultCheckout(true) buildDiscarder(logRotator(numToKeepStr: '3')) } -- GitLab From afc63424b263a455e534d7f5f6f2223a87bab076 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sun, 4 Feb 2024 21:45:14 +0000 Subject: [PATCH 382/447] added checkoutsubdir --- Jenkinsfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e33dbdc4d..54a359a1b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,8 +6,7 @@ pipeline { agent { label 'built-in' } options { - //t disableConcurrentBuilds(abortPrevious: false) - skipDefaultCheckout(true) + skipDefaultCheckout() buildDiscarder(logRotator(numToKeepStr: '3')) } @@ -51,9 +50,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - options { - throttle(['global_matrix_build']) - } + //options { + // throttle(['global_matrix_build']) + //} axes { axis { name "system" @@ -61,6 +60,9 @@ pipeline { } } stages { + options { + checkoutToSubdirectory('${HOME}/${system}') } + } stage("build system") { steps { script { @@ -72,7 +74,6 @@ pipeline { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } else { - deleteDir() checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" -- GitLab From 511ee952d0a26e843817bebec2e3c5c5f96148a4 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sun, 4 Feb 2024 21:48:26 +0000 Subject: [PATCH 383/447] extra bracket --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 54a359a1b..2b84fa3c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { } stages { options { - checkoutToSubdirectory('${HOME}/${system}') } + checkoutToSubdirectory('${HOME}/${system}') } stage("build system") { steps { -- GitLab From 3b3ebc038e0e89f1fa741fddb3391f00764b86cf Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sun, 4 Feb 2024 21:51:03 +0000 Subject: [PATCH 384/447] moved options --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2b84fa3c6..b16edaeed 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -60,10 +60,10 @@ pipeline { } } stages { - options { - checkoutToSubdirectory('${HOME}/${system}') - } stage("build system") { + options { + checkoutToSubdirectory('${HOME}/${system}') + } steps { script { def HOMEgfs = "${HOME}/${system}" -- GitLab From 600cd19024227d7eb5996c782fc66b175620197b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Sun, 4 Feb 2024 22:37:32 +0000 Subject: [PATCH 385/447] add no scm in exp --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index b16edaeed..90b07b98e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -95,6 +95,7 @@ pipeline { stage('Setup RUNTESTS') { agent { label "${MACHINE}-emc" } + options { skipDefaultCheckout() } steps { script { sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) -- GitLab From 297a842a633ed8680ca2400f519e5df1cdae8849 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 00:50:48 +0000 Subject: [PATCH 386/447] devult no scm others --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 90b07b98e..4a2b92765 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -111,6 +111,7 @@ pipeline { stage('Run Tests') { matrix { agent { label "${MACHINE}-emc" } + options { skipDefaultCheckout() } axes { axis { name "Case" @@ -126,7 +127,8 @@ pipeline { system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + sh( script: "echo 'stub create experiment'", returnStatus: true) + //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } -- GitLab From a4174f5b1e8217f540d753a6a097eb2ad72b786d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 01:53:18 +0000 Subject: [PATCH 387/447] put throuttle back in --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4a2b92765..c5f300f6f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,9 +50,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - //options { - // throttle(['global_matrix_build']) - //} + options { + throttle(['global_matrix_build']) + } axes { axis { name "system" -- GitLab From 74dbb16b25f682d048d330d631d86677c29eb8fa Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 14:29:45 +0000 Subject: [PATCH 388/447] try chaning workspace to get scm to checkout on second parallel matrix build --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c5f300f6f..26a913f9c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,6 @@ pipeline { options { skipDefaultCheckout() - buildDiscarder(logRotator(numToKeepStr: '3')) } stages { @@ -68,7 +67,8 @@ pipeline { script { def HOMEgfs = "${HOME}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - dir(HOMEgfs) { + ws(HOMEgfs) { + checkoutToSubdirectory('${HOME}/${system}') env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() -- GitLab From 881697f83fe9dedcffa39b2acf75409ed9e1d8fd Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 14:32:37 +0000 Subject: [PATCH 389/447] added stubs for build all to test the Jenkins scm issue --- ci/cases/yamls/build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 5398fa188..005db938f 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,5 @@ builds: - - gefs: './build_all.sh' - - gfs: './build_all.sh -gu' \ No newline at end of file + #- gefs: './build_all.sh' + #- gfs: './build_all.sh -gu' + - gefs: 'echo "skipping build"' + - gfs: 'echo "skipping build"' \ No newline at end of file -- GitLab From b6f012960188c71890f0d2c84da4423662912bd1 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 14:36:42 +0000 Subject: [PATCH 390/447] removed throttle option for testing scm --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 26a913f9c..dbc969583 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -60,9 +60,9 @@ pipeline { } stages { stage("build system") { - options { - checkoutToSubdirectory('${HOME}/${system}') - } + //options { + // checkoutToSubdirectory('${HOME}/${system}') + //} steps { script { def HOMEgfs = "${HOME}/${system}" -- GitLab From 213f6e450a33e664a20f49586871e1a927b4cecc Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 5 Feb 2024 09:38:47 -0500 Subject: [PATCH 391/447] Update ci/scripts/utils/ci_utils.sh Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov> --- ci/scripts/utils/ci_utils.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 7422d3457..1947bf10b 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -1,5 +1,15 @@ #!/bin/env bash +function determine_scheduler() { + if command -v sbatch &> /dev/null; then + echo "slurm"; + elif command -v qsub &> /dev/null; then + echo "torque"; + else + echo "unknown" + fi +} + function cancel_batch_jobs() { # Usage: cancel_batch_jobs <substring> # Example: cancel_batch_jobs "C48_ATM_3c4e7f74" -- GitLab From 2d7e17b9ebfd6035d490a4d83166d61ea724ab9d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 5 Feb 2024 09:40:02 -0500 Subject: [PATCH 392/447] Update ci/scripts/utils/ci_utils.sh Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov> --- ci/scripts/utils/ci_utils.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 1947bf10b..38caf2e95 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -21,8 +21,9 @@ function cancel_batch_jobs() { local substring=$1 local job_ids - # cancel pbs jobs <substring> - if [[ ${MACHINE_ID} == "wcoss2" ]]; then + scheduler=$(determine_scheduler) + case ${scheduler} in; + "torque") job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true for job_id in ${job_ids}; do -- GitLab From 86722b01692a428fa65e5ed87f9762ac70f579e9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 5 Feb 2024 09:41:53 -0500 Subject: [PATCH 393/447] Update ci/scripts/utils/ci_utils.sh Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov> --- ci/scripts/utils/ci_utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 38caf2e95..ea5c2396b 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -35,7 +35,8 @@ function cancel_batch_jobs() { fi done # cancel slurm jobs <substring> - else + ;; + "torque") job_ids=$(squeue -u "${USER}" -h -o "%i") for job_id in ${job_ids}; do -- GitLab From ef1112f475532a678e10978a27500cc54e84c534 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 5 Feb 2024 09:42:13 -0500 Subject: [PATCH 394/447] Update ci/scripts/utils/ci_utils.sh Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov> --- ci/scripts/utils/ci_utils.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index ea5c2396b..3ab74e9b5 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -47,7 +47,12 @@ function cancel_batch_jobs() { continue fi done - fi + ;; + *) + echo "FATAL: Unknown/unsupported job scheduler" + exit 1 + ;; + esac } -- GitLab From 0448492b71b0b5982b912faf69119a7243108ace Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Mon, 5 Feb 2024 09:43:29 -0500 Subject: [PATCH 395/447] Update modulefiles/module_gwsetup.hera.lua Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov> --- modulefiles/module_gwsetup.hera.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index 244a2419e..961403e1a 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -14,6 +14,7 @@ load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") -load(pathJoin("git","2.40.0")) +local git_ver=os.getenv("git_ver") or "2.40.0" +load(pathJoin("git", git_ver)) whatis("Description: GFS run setup environment") -- GitLab From 62fe05435c1c9646675195effa5628bd4ef63e06 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 14:55:15 +0000 Subject: [PATCH 396/447] took out throuttle and put checkout subdir back --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dbc969583..491a616a0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,9 +49,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - options { - throttle(['global_matrix_build']) - } + //options { + // throttle(['global_matrix_build']) + //} axes { axis { name "system" @@ -60,9 +60,9 @@ pipeline { } stages { stage("build system") { - //options { - // checkoutToSubdirectory('${HOME}/${system}') - //} + options { + checkoutToSubdirectory('${HOME}/${system}') + } steps { script { def HOMEgfs = "${HOME}/${system}" -- GitLab From 45b87fa62c0d793abd1572af20d11905d8ea39cd Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 15:20:30 +0000 Subject: [PATCH 397/447] update build args for faster buld tests --- Jenkinsfile | 3 ++- ci/cases/yamls/build.yaml | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 491a616a0..f3134ab8c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -115,7 +115,8 @@ pipeline { axes { axis { name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 005db938f..753734ced 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,5 +1,3 @@ builds: - #- gefs: './build_all.sh' - #- gfs: './build_all.sh -gu' - - gefs: 'echo "skipping build"' - - gfs: 'echo "skipping build"' \ No newline at end of file + - gefs: './build_all.sh -j 2' + - gfs: './build_all.sh -g -j 2' \ No newline at end of file -- GitLab From e7ecc8f7cca98c6790a4e910b7e0e7996c62536d Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 15:36:02 +0000 Subject: [PATCH 398/447] took stub to create tests and put the action back --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f3134ab8c..5705a12db 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -128,8 +128,7 @@ pipeline { system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "echo 'stub create experiment'", returnStatus: true) - //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } -- GitLab From a238eda58e357da09a54221a3a1c514c5792e8b2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 16:17:15 +0000 Subject: [PATCH 399/447] put throttle back and took out -j 2 --- Jenkinsfile | 6 +++--- ci/cases/yamls/build.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5705a12db..d22d9ea42 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,9 +49,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - //options { - // throttle(['global_matrix_build']) - //} + options { + throttle(['global_matrix_build']) + } axes { axis { name "system" diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 753734ced..38e1b6913 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - - gefs: './build_all.sh -j 2' - - gfs: './build_all.sh -g -j 2' \ No newline at end of file + - gefs: './build_all.sh' + - gfs: './build_all.sh -g' \ No newline at end of file -- GitLab From 7099e77f89fdaa70810effbaac858c84e00fa7cb Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 16:48:34 +0000 Subject: [PATCH 400/447] put concurrancy back in for the matrixed build stage because the scm non reset issue came back --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d22d9ea42..5705a12db 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,9 +49,9 @@ pipeline { stage('Build System') { matrix { agent { label "${MACHINE}-emc" } - options { - throttle(['global_matrix_build']) - } + //options { + // throttle(['global_matrix_build']) + //} axes { axis { name "system" -- GitLab From 25893ae5dfec9516d40d46bef75938bd9bb5a072 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 17:48:47 +0000 Subject: [PATCH 401/447] moved mkdir out of Setup RUNTESTS because it the agent options was not using skepDefaultCheckout and erroring on scm GitHub --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5705a12db..aaf058fde 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" - sh( script: "mkdir -p ${HOME}", returnStatus: true) + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) pullRequest.addLabel("CI-${machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { pullRequest.removeLabel("CI-${machine}-Ready") @@ -98,7 +98,6 @@ pipeline { options { skipDefaultCheckout() } steps { script { - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { pullRequest.removeLabel("CI-${machine}-Building") } -- GitLab From 5ce311631fc30e435265b1d69bd608138a72609c Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 17:58:29 +0000 Subject: [PATCH 402/447] added explcit workspaces in create and using gfs as default for running --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index aaf058fde..3993233df 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -120,7 +120,7 @@ pipeline { } stages { stage('Create Experiment') { - steps { + steps { ws("${HOME}/gfs") { script { sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" @@ -128,11 +128,11 @@ pipeline { def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - } + } } } } stage('Run Experiments') { - steps { + steps { ws("${HOME}/gfs") { script { def HOMEgfs = "${HOME}/gfs" pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() @@ -144,7 +144,7 @@ pipeline { pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") error("Failed to run experiments ${Case} on ${machine}") } - } + } } } } } -- GitLab From 245a5fb59852edfabda840da35af335a20b040d6 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 17:58:38 +0000 Subject: [PATCH 403/447] added explcit workspaces in create and using gfs as default for running --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3993233df..6f1780e07 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -120,15 +120,17 @@ pipeline { } stages { stage('Create Experiment') { - steps { ws("${HOME}/gfs") { + steps { script { sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - } } + ws(HOMEgfs) { + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + } + } } } stage('Run Experiments') { -- GitLab From 0bd51edc7163b1ecc8787975f3be932793d6839f Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 18:16:05 +0000 Subject: [PATCH 404/447] took out case and put in if else in cancel_batch_jobs (for now) --- ci/scripts/utils/ci_utils.sh | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 3ab74e9b5..b0c72a6e9 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -22,8 +22,8 @@ function cancel_batch_jobs() { local job_ids scheduler=$(determine_scheduler) - case ${scheduler} in; - "torque") + + if [[ "${schduler}" == "torque" ]]; then job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true for job_id in ${job_ids}; do @@ -34,9 +34,9 @@ function cancel_batch_jobs() { continue fi done - # cancel slurm jobs <substring> - ;; - "torque") + + elif [[ "${scheduler}" == "slurm" ]]; then + job_ids=$(squeue -u "${USER}" -h -o "%i") for job_id in ${job_ids}; do @@ -47,13 +47,11 @@ function cancel_batch_jobs() { continue fi done - ;; - *) - echo "FATAL: Unknown/unsupported job scheduler" - exit 1 - ;; - esac -} + + else + echo "FATAL: Unknown/unsupported job scheduler" + exit 1 + fi function get_pr_case_list () { -- GitLab From cc6af4257fa44b16e0710b8ac8b585959c92d133 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 18:38:24 +0000 Subject: [PATCH 405/447] missed bracket at end of cancel_batch_jobs in ci utils --- ci/scripts/utils/ci_utils.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index b0c72a6e9..6f2426c38 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -52,6 +52,7 @@ function cancel_batch_jobs() { echo "FATAL: Unknown/unsupported job scheduler" exit 1 fi +} function get_pr_case_list () { -- GitLab From 5c5e860b97a032134ca2e88ec5f3b2c90217cff2 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 20:14:10 +0000 Subject: [PATCH 406/447] add true to return status on run-check because it through that peski Jobs SCM is not GitHub again and it is the only difference to the lines that did not --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6f1780e07..b5228f360 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -128,7 +128,8 @@ pipeline { def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" ws(HOMEgfs) { - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + sh( script: "echo 'skip create_experiment'" , returnStatus: true) } } } @@ -140,7 +141,7 @@ pipeline { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") try { - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false) + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } catch (Exception e) { pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") -- GitLab From 743e8424dc0aa6f0c45cef0e93446d37c209f94b Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 20:24:06 +0000 Subject: [PATCH 407/447] update all Jenkins script calls to returnStatus true and put full test list back --- Jenkinsfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5228f360..4d981d45c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,8 +80,8 @@ pipeline { def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: false) - sh( script: "./link_workflow.sh", returnStatus: false) + sh( script: "${build_args}", returnStatus: true) + sh( script: "./link_workflow.sh", returnStatus: true) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } @@ -114,8 +114,7 @@ pipeline { axes { axis { name "Case" - //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } stages { @@ -128,8 +127,7 @@ pipeline { def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" ws(HOMEgfs) { - //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - sh( script: "echo 'skip create_experiment'" , returnStatus: true) + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } -- GitLab From 01298bb834952ddaa487c10b64363a98b6ab0572 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:28:47 +0000 Subject: [PATCH 408/447] adding with credentials --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 4d981d45c..a2ed5a50f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,11 +157,13 @@ pipeline { post { always { script { + withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { for (label in pullRequest.labels) { if (label.contains("${machine}")) { pullRequest.removeLabel(label) } } + } } } success { -- GitLab From 0a32c5b0e1cad8b9e9f7cc7edefefd9841a56ae8 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:31:15 +0000 Subject: [PATCH 409/447] stub create --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a2ed5a50f..7906c0c75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -127,7 +127,8 @@ pipeline { def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" ws(HOMEgfs) { - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + sh( script: "echo 'skip create", returnStatus: true) } } } -- GitLab From 5bf9d34b35d7d80c206991cfd6f4d41f9c7534d7 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:34:20 +0000 Subject: [PATCH 410/447] add short list for exper --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7906c0c75..bfd3dfb90 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,7 +114,8 @@ pipeline { axes { axis { name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { -- GitLab From b74d9826bfe5c510977872f8df5f6c8dd3842842 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:48:25 +0000 Subject: [PATCH 411/447] added withCredntials in all three post sections --- Jenkinsfile | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bfd3dfb90..3f2ec815e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -160,31 +160,35 @@ pipeline { always { script { withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { - for (label in pullRequest.labels) { - if (label.contains("${machine}")) { - pullRequest.removeLabel(label) + for (label in pullRequest.labels) { + if (label.contains("${machine}")) { + pullRequest.removeLabel(label) + } } } - } } } success { script { - pullRequest.addLabel("CI-${machine}-Passed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") + withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { + pullRequest.addLabel("CI-${machine}-Passed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") + } } } failure { script { - pullRequest.addLabel("CI-${machine}-Failed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") - if (fileExists('${HOME}/RUNTESTS/ci.log')) { - def fileContent = readFile '${HOME}/RUNTESTS/ci.log' - fileContent.eachLine { line -> - if( line.contains(".log")) { - archiveArtifacts artifacts: "${line}", fingerprint: true + withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { + pullRequest.addLabel("CI-${machine}-Failed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + if( line.contains(".log")) { + archiveArtifacts artifacts: "${line}", fingerprint: true + } } } } -- GitLab From 04b3ed8138d6ac9b665eaabd0025bb5b96113a2e Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:53:43 +0000 Subject: [PATCH 412/447] took out create experments stub --- Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3f2ec815e..f44dea693 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,8 +114,8 @@ pipeline { axes { axis { name "Case" - //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" + // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { @@ -128,8 +128,7 @@ pipeline { def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" ws(HOMEgfs) { - //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - sh( script: "echo 'skip create", returnStatus: true) + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } -- GitLab From 0f9605691f0002238bff782048f6d87cc91469c3 Mon Sep 17 00:00:00 2001 From: tmcguinness <terry.mcguinness@noaa.gov> Date: Mon, 5 Feb 2024 22:56:09 +0000 Subject: [PATCH 413/447] put back in full test list for overnight run --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f44dea693..f905004df 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,8 +114,7 @@ pipeline { axes { axis { name "Case" - // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } stages { -- GitLab From 271a453b3ae4fc213cf91f5d51911994d19f41b9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Tue, 6 Feb 2024 00:50:17 -0500 Subject: [PATCH 414/447] Update Jenkinsfile removed post because withCrednitlals failed on first pass --- Jenkinsfile | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f905004df..a182a3827 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,29 +155,19 @@ pipeline { } post { - always { - script { - withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { - for (label in pullRequest.labels) { - if (label.contains("${machine}")) { - pullRequest.removeLabel(label) - } - } - } - } - } + success { script { - withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { + // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") } - } + //} } failure { script { - withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { + // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") @@ -189,7 +179,7 @@ pipeline { } } } - } + //} } } } -- GitLab From 9bd850b94fb164ce1a3ef5129cd360cd4882d089 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 12:47:21 -0500 Subject: [PATCH 415/447] add agent to post block --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a182a3827..5cff08354 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,7 +155,7 @@ pipeline { } post { - + agent { label "${MACHINE}-emc" } success { script { // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { -- GitLab From 343c8df9cfdfd96e7ad064bb99075c640a636a7f Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 13:37:15 -0500 Subject: [PATCH 416/447] move post into Run Tests block --- Jenkinsfile | 67 +++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5cff08354..ee938d1fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -110,7 +110,6 @@ pipeline { stage('Run Tests') { matrix { agent { label "${MACHINE}-emc" } - options { skipDefaultCheckout() } axes { axis { name "Case" @@ -133,7 +132,7 @@ pipeline { } } stage('Run Experiments') { - steps { ws("${HOME}/gfs") { + steps { script { def HOMEgfs = "${HOME}/gfs" pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() @@ -145,41 +144,43 @@ pipeline { pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") error("Failed to run experiments ${Case} on ${machine}") } - } } + } } - } - } - } - } - - } - - post { - agent { label "${MACHINE}-emc" } - success { - script { - // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { - pullRequest.addLabel("CI-${machine}-Passed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}") - } - //} - } - failure { - script { - // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) { - pullRequest.addLabel("CI-${machine}-Failed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") - if (fileExists('${HOME}/RUNTESTS/ci.log')) { - def fileContent = readFile '${HOME}/RUNTESTS/ci.log' - fileContent.eachLine { line -> - if( line.contains(".log")) { - archiveArtifacts artifacts: "${line}", fingerprint: true + post { + always { + script { + for (label in pullRequest.labels) { + if (label.contains("${machine}")) { + pullRequest.removeLabel(label) + } + } + } + } + success { + script { + pullRequest.addLabel("CI-${machine}-Success") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + } + } + failure { + script { + pullRequest.addLabel("CI-${machine}-Failed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + if( line.contains(".log")) { + archiveArtifacts artifacts: "${line}", fingerprint: true + } + } + } + } } } } - //} + } } } } -- GitLab From a57e4af50c58b04edf90787bf267698cbfdcc5ab Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 13:38:54 -0500 Subject: [PATCH 417/447] stubbed out create expermentens again --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ee938d1fe..8ca7e69f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -125,9 +125,9 @@ pipeline { system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - ws(HOMEgfs) { - sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - } + //ws(HOMEgfs) { + // sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + //} } } } -- GitLab From 967bb6ddb8f1dc9cfe26374367abb4482f73c14e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 13:45:17 -0500 Subject: [PATCH 418/447] took out sub for create and made case short list --- Jenkinsfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ca7e69f8..ca38da181 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,8 @@ pipeline { axes { axis { name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { @@ -125,9 +126,7 @@ pipeline { system = yaml_case.experiment.system def HOMEgfs = "${HOME}/${system}" env.RUNTESTS = "${HOME}/RUNTESTS" - //ws(HOMEgfs) { - // sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - //} + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } } } @@ -158,7 +157,7 @@ pipeline { } success { script { - pullRequest.addLabel("CI-${machine}-Success") + pullRequest.addLabel("CI-${machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") } -- GitLab From 0ec0b6f0348d4395d23a54789960d5724ca62852 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 15:22:34 -0500 Subject: [PATCH 419/447] removed setup block --- Jenkinsfile | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca38da181..f73b418e9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,6 +86,10 @@ pipeline { } } } + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { + pullRequest.removeLabel("CI-${machine}-Building") + } + pullRequest.addLabel("CI-${machine}-Running") } } } @@ -93,20 +97,6 @@ pipeline { } } - stage('Setup RUNTESTS') { - agent { label "${MACHINE}-emc" } - options { skipDefaultCheckout() } - steps { - script { - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { - pullRequest.removeLabel("CI-${machine}-Building") - } - pullRequest.addLabel("CI-${machine}-Running") - } - } - - } - stage('Run Tests') { matrix { agent { label "${MACHINE}-emc" } -- GitLab From 5914683a69e5dac4d74de271909a109571545cb2 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 15:28:20 -0500 Subject: [PATCH 420/447] just do one test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f73b418e9..31df1fa1a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -104,7 +104,7 @@ pipeline { axis { name "Case" // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" + values "C48_S2SWA_gefs" } } stages { -- GitLab From 9b837357e80fa4db6dd86baaf3a9c4d990e4c828 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 15:31:48 -0500 Subject: [PATCH 421/447] added back buildDiscarder --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 31df1fa1a..371ee930a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,6 +7,7 @@ pipeline { options { skipDefaultCheckout() + buildDiscarder(logRotator(numToKeepStr: '2')) } stages { -- GitLab From 970e73b2ddc61c9ce51d748cd993e51ce28205ec Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 19:15:54 -0500 Subject: [PATCH 422/447] commented out remove label in build --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 371ee930a..28848291e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,9 +87,9 @@ pipeline { } } } - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { - pullRequest.removeLabel("CI-${machine}-Building") - } + //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { + // pullRequest.removeLabel("CI-${machine}-Building") + //} pullRequest.addLabel("CI-${machine}-Running") } } -- GitLab From 23fad4c44659419f40033371bdcd8f91964519ee Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 20:15:29 -0500 Subject: [PATCH 423/447] removed redundent checkout in subdir and commentout gitHub access after scm in build --- Jenkinsfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28848291e..fa22a6ae1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,15 +61,11 @@ pipeline { } stages { stage("build system") { - options { - checkoutToSubdirectory('${HOME}/${system}') - } steps { script { def HOMEgfs = "${HOME}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) ws(HOMEgfs) { - checkoutToSubdirectory('${HOME}/${system}') env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() @@ -90,7 +86,7 @@ pipeline { //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { // pullRequest.removeLabel("CI-${machine}-Building") //} - pullRequest.addLabel("CI-${machine}-Running") + //pullRequest.addLabel("CI-${machine}-Running") } } } -- GitLab From 4b8bec7e9d249298a6cf16d042ce13fb19308784 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 22:42:28 -0500 Subject: [PATCH 424/447] added ws with github creditials --- Jenkinsfile | 70 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fa22a6ae1..43610c9a0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,7 @@ def MACHINE = 'none' def machine = 'none' def HOME = 'none' +def localworkspace = 'none' pipeline { agent { label 'built-in' } @@ -16,32 +17,34 @@ pipeline { agent { label 'built-in' } steps { script { - MACHINE = 'none' + localworkspace = env.WORKSPACE + machine = 'none' for (label in pullRequest.labels) { echo "Label: ${label}" if ((label.matches("CI-Hera-Ready"))) { - MACHINE = 'hera' + machine = 'hera' } else if ((label.matches("CI-Orion-Ready"))) { - MACHINE = 'orion' + machine = 'orion' } else if ((label.matches("CI-Hercules-Ready"))) { - MACHINE = 'hercules' + machine = 'hercules' } } - machine = MACHINE[0].toUpperCase() + MACHINE.substring(1) + Machine = machine[0].toUpperCase() + machine.substring(1) } } } stage('Get Common Workspace') { - agent { label "${MACHINE}-emc" } + agent { label "${machine}-emc" } steps ( timeout(time: 1, unit: 'HOURS') ) { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" + commonworkspace = "${WORKSPCE}" sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - pullRequest.addLabel("CI-${machine}-Building") - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) { - pullRequest.removeLabel("CI-${machine}-Ready") + pullRequest.addLabel("CI-${Machine}-Building") + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) { + pullRequest.removeLabel("CI-${Machine}-Ready") } } } @@ -69,7 +72,7 @@ pipeline { env.MACHINE_ID = MACHINE if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() - pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") + ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } } else { checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) @@ -82,11 +85,12 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } + + if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { + pullRequest.removeLabel("CI-${machine}-Building") + } + pullRequest.addLabel("CI-${machine}-Running") } - //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { - // pullRequest.removeLabel("CI-${machine}-Building") - //} - //pullRequest.addLabel("CI-${machine}-Running") } } } @@ -101,7 +105,7 @@ pipeline { axis { name "Case" // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { @@ -121,36 +125,43 @@ pipeline { steps { script { def HOMEgfs = "${HOME}/gfs" - pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") - try { - sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") - } catch (Exception e) { - pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") - error("Failed to run experiments ${Case} on ${machine}") + ws(HOMEgfs) { + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + try { + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + } catch (Exception e) { + pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") + error("Failed to run experiments ${Case} on ${machine}") + } } } } post { always { script { - for (label in pullRequest.labels) { - if (label.contains("${machine}")) { - pullRequest.removeLabel(label) + ws (localworkspace) { + for (label in pullRequest.labels) { + if (label.contains("${machine}")) { + pullRequest.removeLabel(label) + } } } } } success { script { - pullRequest.addLabel("CI-${machine}-Passed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + ws (localwospace) { + pullRequest.addLabel("CI-${machine}-Passed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + } } } failure { script { + ws (localworkspace) { pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") @@ -162,6 +173,7 @@ pipeline { } } } + } } } } -- GitLab From db37fe0a849544652e949657636f2f67094dfe60 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 22:46:13 -0500 Subject: [PATCH 425/447] misspelled WORKSPACE --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 43610c9a0..762387124 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" - commonworkspace = "${WORKSPCE}" + commonworkspace = "${WORKSPACE}" sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) pullRequest.addLabel("CI-${Machine}-Building") if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) { -- GitLab From 682186dc4652caea02e83ec983fa9b869b602fa0 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Tue, 6 Feb 2024 22:53:48 -0500 Subject: [PATCH 426/447] missed a MACIHNE left in --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 762387124..c4f2c3aa2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -def MACHINE = 'none' +def Machine = 'none' def machine = 'none' def HOME = 'none' def localworkspace = 'none' @@ -52,7 +52,7 @@ pipeline { stage('Build System') { matrix { - agent { label "${MACHINE}-emc" } + agent { label "${machine}-emc" } //options { // throttle(['global_matrix_build']) //} @@ -69,7 +69,7 @@ pipeline { def HOMEgfs = "${HOME}/${system}" sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) ws(HOMEgfs) { - env.MACHINE_ID = MACHINE + env.MACHINE_ID = machine if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } @@ -100,7 +100,7 @@ pipeline { stage('Run Tests') { matrix { - agent { label "${MACHINE}-emc" } + agent { label "${machine}-emc" } axes { axis { name "Case" -- GitLab From 8aa8b026a5a677fdb8524f74c54e9da93e33a3f6 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 09:50:37 -0500 Subject: [PATCH 427/447] took timer out of Get Common --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c4f2c3aa2..3c95a1c91 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,6 +2,7 @@ def Machine = 'none' def machine = 'none' def HOME = 'none' def localworkspace = 'none' +def commonworkspace = 'none' pipeline { agent { label 'built-in' } @@ -36,7 +37,7 @@ pipeline { stage('Get Common Workspace') { agent { label "${machine}-emc" } - steps ( timeout(time: 1, unit: 'HOURS') ) { + steps { script { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}/TESTDIR" @@ -72,7 +73,7 @@ pipeline { env.MACHINE_ID = machine if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() - ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } + ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } } else { checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) -- GitLab From ed0562b58a67f1e28017b0605e27d0edf13f892d Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 10:50:38 -0500 Subject: [PATCH 428/447] updated machine names to Machine in labels missed --- Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c95a1c91..fe71778f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,10 +87,10 @@ pipeline { } } - if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) { - pullRequest.removeLabel("CI-${machine}-Building") + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { + pullRequest.removeLabel("CI-${Machine}-Building") } - pullRequest.addLabel("CI-${machine}-Running") + pullRequest.addLabel("CI-${Machine}-Running") } } } @@ -144,7 +144,7 @@ pipeline { script { ws (localworkspace) { for (label in pullRequest.labels) { - if (label.contains("${machine}")) { + if (label.contains("${Machine}")) { pullRequest.removeLabel(label) } } @@ -153,10 +153,10 @@ pipeline { } success { script { - ws (localwospace) { - pullRequest.addLabel("CI-${machine}-Passed") + ws (localworkspace) { + pullRequest.addLabel("CI-${Machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") } } } @@ -165,7 +165,7 @@ pipeline { ws (localworkspace) { pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") if (fileExists('${HOME}/RUNTESTS/ci.log')) { def fileContent = readFile '${HOME}/RUNTESTS/ci.log' fileContent.eachLine { line -> -- GitLab From d94a873ab4cb0502a0915d6700137e0d71c2997e Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 11:29:22 -0500 Subject: [PATCH 429/447] added commonworkspace in build update labels --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe71778f6..057c40ce3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,11 +86,12 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } - + ws(commonworkspace) { if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { pullRequest.removeLabel("CI-${Machine}-Building") } pullRequest.addLabel("CI-${Machine}-Running") + } } } } -- GitLab From 5c04148c7798182add5b7201d85bfb7fd206dc3e Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 14:56:59 -0500 Subject: [PATCH 430/447] updated workspace in post to be the same in the matrix that ran the scripts and should have a prevouse scm checkout profomed in it --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 057c40ce3..7f75a04a0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -132,18 +132,18 @@ pipeline { pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } catch (Exception e) { pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") error("Failed to run experiments ${Case} on ${machine}") } + pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") } } } post { always { script { - ws (localworkspace) { + ws (HOMEgfs) { for (label in pullRequest.labels) { if (label.contains("${Machine}")) { pullRequest.removeLabel(label) @@ -154,7 +154,7 @@ pipeline { } success { script { - ws (localworkspace) { + ws (HOMEgfs) { pullRequest.addLabel("CI-${Machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") @@ -163,7 +163,7 @@ pipeline { } failure { script { - ws (localworkspace) { + ws (HOMEgfs) { pullRequest.addLabel("CI-${machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") -- GitLab From 8cf8a2af70b462b9c56acabd5ab3ec66937eec41 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 15:10:06 -0500 Subject: [PATCH 431/447] remove local varible scope on HOMEgfs in Run experments in hopes it works in post --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7f75a04a0..3852fbcc9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,7 +117,7 @@ pipeline { sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" system = yaml_case.experiment.system - def HOMEgfs = "${HOME}/${system}" + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis env.RUNTESTS = "${HOME}/RUNTESTS" sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) } @@ -126,7 +126,7 @@ pipeline { stage('Run Experiments') { steps { script { - def HOMEgfs = "${HOME}/gfs" + HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") -- GitLab From 196424704910ec60f6161d38574263fc4740be94 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 15:35:27 -0500 Subject: [PATCH 432/447] updated Machine in a label and in PR comments --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3852fbcc9..9c51a7d22 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,14 +129,14 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}") + pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${Machine}") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) } catch (Exception e) { - pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}") - error("Failed to run experiments ${Case} on ${machine}") + pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}") + error("Failed to run experiments ${Case} on ${Machine}") } - pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") + pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}") } } } @@ -164,7 +164,7 @@ pipeline { failure { script { ws (HOMEgfs) { - pullRequest.addLabel("CI-${machine}-Failed") + pullRequest.addLabel("CI-${Machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") if (fileExists('${HOME}/RUNTESTS/ci.log')) { -- GitLab From b2c3c316c03d34f53d6fe8b759e4e50861d8db56 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 16:02:18 -0500 Subject: [PATCH 433/447] only replaced commonworkspace with HOMEgfs workspace in build matrix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9c51a7d22..4678299cd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } - ws(commonworkspace) { + ws(HOMEgfs) { if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { pullRequest.removeLabel("CI-${Machine}-Building") } -- GitLab From 12ec3627739743ea453ed5270bdc88bf846e254f Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 17:04:23 -0500 Subject: [PATCH 434/447] added info in Running message to use in GitHub PR --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4678299cd..85dbbf285 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,7 +107,6 @@ pipeline { axis { name "Case" // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { @@ -129,7 +128,7 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${Machine}") + pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) } catch (Exception e) { -- GitLab From eb9e48fb3d62465e8cd883bf9509057f316cb327 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 17:10:09 -0500 Subject: [PATCH 435/447] put missing values line back in experments matrix --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 85dbbf285..355a4476f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,6 +107,7 @@ pipeline { axis { name "Case" // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_S2SWA_gefs", "C96_atm3DVar" } } stages { -- GitLab From b0dab475c1c2a7639e821a276e76b03e91defef4 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 17:27:29 -0500 Subject: [PATCH 436/447] added u to gfs build and put all tests back in --- Jenkinsfile | 3 +-- ci/cases/yamls/build.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 355a4476f..57a09fc2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,8 +106,7 @@ pipeline { axes { axis { name "Case" - // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - values "C48_S2SWA_gefs", "C96_atm3DVar" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" } } stages { diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml index 38e1b6913..5398fa188 100644 --- a/ci/cases/yamls/build.yaml +++ b/ci/cases/yamls/build.yaml @@ -1,3 +1,3 @@ builds: - gefs: './build_all.sh' - - gfs: './build_all.sh -g' \ No newline at end of file + - gfs: './build_all.sh -gu' \ No newline at end of file -- GitLab From 9bb5c97173aa9433fc9203ee1d00d4879dcb829b Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 17:56:25 -0500 Subject: [PATCH 437/447] added a stubbed version of the Jenkinsfiel for testing --- Jenkinsfile_stubbed | 188 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 Jenkinsfile_stubbed diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed new file mode 100644 index 000000000..211c87324 --- /dev/null +++ b/Jenkinsfile_stubbed @@ -0,0 +1,188 @@ +def Machine = 'none' +def machine = 'none' +def HOME = 'none' +def localworkspace = 'none' +def commonworkspace = 'none' + +pipeline { + agent { label 'built-in' } + + options { + skipDefaultCheckout() + buildDiscarder(logRotator(numToKeepStr: '2')) + } + + stages { + + stage('Get Machine') { + agent { label 'built-in' } + steps { + script { + localworkspace = env.WORKSPACE + machine = 'none' + for (label in pullRequest.labels) { + echo "Label: ${label}" + if ((label.matches("CI-Hera-Ready"))) { + machine = 'hera' + } else if ((label.matches("CI-Orion-Ready"))) { + machine = 'orion' + } else if ((label.matches("CI-Hercules-Ready"))) { + machine = 'hercules' + } + } + Machine = machine[0].toUpperCase() + machine.substring(1) + } + } + } + + stage('Get Common Workspace') { + agent { label "${machine}-emc" } + steps { + script { + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + HOME = "${WORKSPACE}/TESTDIR" + commonworkspace = "${WORKSPACE}" + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) + pullRequest.addLabel("CI-${Machine}-Building") + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) { + pullRequest.removeLabel("CI-${Machine}-Ready") + } + } + } + } + + stage('Build System') { + matrix { + agent { label "${machine}-emc" } + //options { + // throttle(['global_matrix_build']) + //} + axes { + axis { + name "system" + values "gfs", "gefs" + } + } + stages { + stage("build system") { + steps { + script { + def HOMEgfs = "${HOME}/${system}" + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) + ws(HOMEgfs) { + env.MACHINE_ID = machine + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { + sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() + ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } + } else { + checkout scm + sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" + def build_args_list = builds_file['builds'] + def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") + dir("${HOMEgfs}/sorc") { + //sh( script: "${build_args}", returnStatus: true) + //sh( script: "./link_workflow.sh", returnStatus: true) + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + } + } + ws(HOMEgfs) { + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { + pullRequest.removeLabel("CI-${Machine}-Building") + } + pullRequest.addLabel("CI-${Machine}-Running") + } + } + } + } + } + } + } + } + + stage('Run Tests') { + matrix { + agent { label "${machine}-emc" } + axes { + axis { + name "Case" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + } + } + stages { + stage('Create Experiment') { + steps { + script { + sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" + system = yaml_case.experiment.system + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.RUNTESTS = "${HOME}/RUNTESTS" + //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) + } + } + } + stage('Run Experiments') { + steps { + script { + HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + ws(HOMEgfs) { + //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() + pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}") + try { + //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) + sh( script: "echo 'subbed run-check'", returnStatus: true) + } catch (Exception e) { + pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}") + error("Failed to run experiments ${Case} on ${Machine}") + } + pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}") + } + } + } + post { + always { + script { + ws (HOMEgfs) { + for (label in pullRequest.labels) { + if (label.contains("${Machine}")) { + pullRequest.removeLabel(label) + } + } + } + } + } + success { + script { + ws (HOMEgfs) { + pullRequest.addLabel("CI-${Machine}-Passed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + } + } + } + failure { + script { + ws (HOMEgfs) { + pullRequest.addLabel("CI-${Machine}-Failed") + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) + pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + if (fileExists('${HOME}/RUNTESTS/ci.log')) { + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' + fileContent.eachLine { line -> + if( line.contains(".log")) { + archiveArtifacts artifacts: "${line}", fingerprint: true + } + } + } + } + } + } + } + } + } + } + } + } + +} -- GitLab From 44947e092d92a0429c09e337e8faeb2b39fd7284 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 19:34:26 -0500 Subject: [PATCH 438/447] put a try around removelabel in all --- Jenkinsfile_stubbed | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed index 211c87324..5d1f23efa 100644 --- a/Jenkinsfile_stubbed +++ b/Jenkinsfile_stubbed @@ -146,7 +146,11 @@ pipeline { ws (HOMEgfs) { for (label in pullRequest.labels) { if (label.contains("${Machine}")) { - pullRequest.removeLabel(label) + try { + pullRequest.removeLabel(label) + } catch (Exception e) { + echo "Failed to remove label ${label}" + } } } } -- GitLab From 0f0df6297c7a3a2ebab34b6ef0486e18860cc9ca Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 19:45:15 -0500 Subject: [PATCH 439/447] took pslot out for stubbing --- Jenkinsfile_stubbed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed index 5d1f23efa..efec58b44 100644 --- a/Jenkinsfile_stubbed +++ b/Jenkinsfile_stubbed @@ -128,7 +128,7 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}") + pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS") try { //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) sh( script: "echo 'subbed run-check'", returnStatus: true) -- GitLab From 562989057ec3bce49713ab6a731bef2ed2f8d792 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 20:07:51 -0500 Subject: [PATCH 440/447] added back just build to stub --- Jenkinsfile_stubbed | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed index efec58b44..b64584189 100644 --- a/Jenkinsfile_stubbed +++ b/Jenkinsfile_stubbed @@ -71,21 +71,21 @@ pipeline { sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) ws(HOMEgfs) { env.MACHINE_ID = machine - if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { - sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() - ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } - } else { + //if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { + // sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() + // ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } + //} else { checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") dir("${HOMEgfs}/sorc") { - //sh( script: "${build_args}", returnStatus: true) - //sh( script: "./link_workflow.sh", returnStatus: true) + sh( script: "${build_args}", returnStatus: true) + sh( script: "./link_workflow.sh", returnStatus: true) sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } - } + //} ws(HOMEgfs) { if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { pullRequest.removeLabel("CI-${Machine}-Building") -- GitLab From 89954ddc39beddc4cafb152adadda7c959ed80ff Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 21:36:07 -0500 Subject: [PATCH 441/447] removed build non stub and made stub again --- Jenkinsfile_stubbed | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed index b64584189..363bb10cc 100644 --- a/Jenkinsfile_stubbed +++ b/Jenkinsfile_stubbed @@ -71,21 +71,22 @@ pipeline { sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) ws(HOMEgfs) { env.MACHINE_ID = machine - //if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { - // sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() - // ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } - //} else { + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { + sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() + ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } + } else { checkout scm sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) def builds_file = readYaml file: "ci/cases/yamls/build.yaml" def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: true) - sh( script: "./link_workflow.sh", returnStatus: true) - sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + //sh( script: "${build_args}", returnStatus: true) + //sh( script: "./link_workflow.sh", returnStatus: true) + //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) + sh( script: "echo 'stubbing build scripts'", returnStatus: true) } - //} + } ws(HOMEgfs) { if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { pullRequest.removeLabel("CI-${Machine}-Building") @@ -128,7 +129,7 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS") + pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/") try { //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) sh( script: "echo 'subbed run-check'", returnStatus: true) -- GitLab From 1ae8f43b43f16ce424c474ba83fcb1c9c127d2aa Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 21:57:54 -0500 Subject: [PATCH 442/447] added only buils scripts but disabled otherwise scan poll --- Jenkinsfile_stubbed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed index 363bb10cc..09fad6675 100644 --- a/Jenkinsfile_stubbed +++ b/Jenkinsfile_stubbed @@ -81,10 +81,10 @@ pipeline { def build_args_list = builds_file['builds'] def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") dir("${HOMEgfs}/sorc") { - //sh( script: "${build_args}", returnStatus: true) - //sh( script: "./link_workflow.sh", returnStatus: true) + sh( script: "${build_args}", returnStatus: true) + sh( script: "./link_workflow.sh", returnStatus: true) //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - sh( script: "echo 'stubbing build scripts'", returnStatus: true) + sh( script: "echo 'ran with build scripts on'", returnStatus: true) } } ws(HOMEgfs) { -- GitLab From 3c83d6f15809b1f5974a9cd5996b2b1940e433e9 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Wed, 7 Feb 2024 23:29:21 -0500 Subject: [PATCH 443/447] remove nested ws in build --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 57a09fc2c..f851b5319 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,12 +86,10 @@ pipeline { sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) } } - ws(HOMEgfs) { if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { pullRequest.removeLabel("CI-${Machine}-Building") } pullRequest.addLabel("CI-${Machine}-Running") - } } } } -- GitLab From 2338e9a84d25dc925814eb988de2b9fbd2f65ea5 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Thu, 8 Feb 2024 12:19:33 -0500 Subject: [PATCH 444/447] added comments in Jenkinsfile --- Jenkinsfile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f851b5319..0ada7682d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,9 @@ pipeline { buildDiscarder(logRotator(numToKeepStr: '2')) } - stages { + stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR + // which is used to designate the Nodes in the Jenkins Controler by the agent label + // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel stage('Get Machine') { agent { label 'built-in' } @@ -29,7 +31,8 @@ pipeline { } else if ((label.matches("CI-Hercules-Ready"))) { machine = 'hercules' } - } + } // createing a second machine varible with first letter capital + // because the first letter of the machine name is captitalized in the GitHub labels Machine = machine[0].toUpperCase() + machine.substring(1) } } @@ -67,12 +70,12 @@ pipeline { stage("build system") { steps { script { - def HOMEgfs = "${HOME}/${system}" + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) ws(HOMEgfs) { - env.MACHINE_ID = machine - if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { - sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() + env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs + sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } } else { checkout scm @@ -104,7 +107,7 @@ pipeline { axes { axis { name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" // TODO add dynamic list of cases from env vars (needs addtional plugins) } } stages { -- GitLab From 62ef2a419ffa15632f3ad56787d674e69c37dffb Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Thu, 8 Feb 2024 14:06:00 -0500 Subject: [PATCH 445/447] added some markdown formatting in GitHub messages --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0ada7682d..0d655d823 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,14 +129,14 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}") + pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}`") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) } catch (Exception e) { - pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}") + pullRequest.comment("**FAILURE** running experiments: ${Case} on ${Machine}") error("Failed to run experiments ${Case} on ${Machine}") } - pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}") + pullRequest.comment("**SUCCESS** running experiments: ${Case} on ${Machine}") } } } @@ -157,7 +157,7 @@ pipeline { ws (HOMEgfs) { pullRequest.addLabel("CI-${Machine}-Passed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`") } } } @@ -166,7 +166,7 @@ pipeline { ws (HOMEgfs) { pullRequest.addLabel("CI-${Machine}-Failed") def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") + pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`") if (fileExists('${HOME}/RUNTESTS/ci.log')) { def fileContent = readFile '${HOME}/RUNTESTS/ci.log' fileContent.eachLine { line -> -- GitLab From cbc1bbe0caaa10f653ae01ec9913ae91b5e14939 Mon Sep 17 00:00:00 2001 From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov> Date: Thu, 8 Feb 2024 14:18:32 -0500 Subject: [PATCH 446/447] updated some markdown formatting in main GitHub PR message --- Jenkinsfile | 2 +- Jenkinsfile_stubbed | 193 -------------------------------------------- 2 files changed, 1 insertion(+), 194 deletions(-) delete mode 100644 Jenkinsfile_stubbed diff --git a/Jenkinsfile b/Jenkinsfile index 0d655d823..0d7c88a96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}`") + pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory `${HOME}/RUNTESTS/${pslot}`") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) } catch (Exception e) { diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed deleted file mode 100644 index 09fad6675..000000000 --- a/Jenkinsfile_stubbed +++ /dev/null @@ -1,193 +0,0 @@ -def Machine = 'none' -def machine = 'none' -def HOME = 'none' -def localworkspace = 'none' -def commonworkspace = 'none' - -pipeline { - agent { label 'built-in' } - - options { - skipDefaultCheckout() - buildDiscarder(logRotator(numToKeepStr: '2')) - } - - stages { - - stage('Get Machine') { - agent { label 'built-in' } - steps { - script { - localworkspace = env.WORKSPACE - machine = 'none' - for (label in pullRequest.labels) { - echo "Label: ${label}" - if ((label.matches("CI-Hera-Ready"))) { - machine = 'hera' - } else if ((label.matches("CI-Orion-Ready"))) { - machine = 'orion' - } else if ((label.matches("CI-Hercules-Ready"))) { - machine = 'hercules' - } - } - Machine = machine[0].toUpperCase() + machine.substring(1) - } - } - } - - stage('Get Common Workspace') { - agent { label "${machine}-emc" } - steps { - script { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) - HOME = "${WORKSPACE}/TESTDIR" - commonworkspace = "${WORKSPACE}" - sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) - pullRequest.addLabel("CI-${Machine}-Building") - if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) { - pullRequest.removeLabel("CI-${Machine}-Ready") - } - } - } - } - - stage('Build System') { - matrix { - agent { label "${machine}-emc" } - //options { - // throttle(['global_matrix_build']) - //} - axes { - axis { - name "system" - values "gfs", "gefs" - } - } - stages { - stage("build system") { - steps { - script { - def HOMEgfs = "${HOME}/${system}" - sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) - ws(HOMEgfs) { - env.MACHINE_ID = machine - if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { - sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() - ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } - } else { - checkout scm - sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) - def builds_file = readYaml file: "ci/cases/yamls/build.yaml" - def build_args_list = builds_file['builds'] - def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") - dir("${HOMEgfs}/sorc") { - sh( script: "${build_args}", returnStatus: true) - sh( script: "./link_workflow.sh", returnStatus: true) - //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) - sh( script: "echo 'ran with build scripts on'", returnStatus: true) - } - } - ws(HOMEgfs) { - if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { - pullRequest.removeLabel("CI-${Machine}-Building") - } - pullRequest.addLabel("CI-${Machine}-Running") - } - } - } - } - } - } - } - } - - stage('Run Tests') { - matrix { - agent { label "${machine}-emc" } - axes { - axis { - name "Case" - values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" - } - } - stages { - stage('Create Experiment') { - steps { - script { - sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) - def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" - system = yaml_case.experiment.system - def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.RUNTESTS = "${HOME}/RUNTESTS" - //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) - } - } - } - stage('Run Experiments') { - steps { - script { - HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments - ws(HOMEgfs) { - //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/") - try { - //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) - sh( script: "echo 'subbed run-check'", returnStatus: true) - } catch (Exception e) { - pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}") - error("Failed to run experiments ${Case} on ${Machine}") - } - pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}") - } - } - } - post { - always { - script { - ws (HOMEgfs) { - for (label in pullRequest.labels) { - if (label.contains("${Machine}")) { - try { - pullRequest.removeLabel(label) - } catch (Exception e) { - echo "Failed to remove label ${label}" - } - } - } - } - } - } - success { - script { - ws (HOMEgfs) { - pullRequest.addLabel("CI-${Machine}-Passed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") - } - } - } - failure { - script { - ws (HOMEgfs) { - pullRequest.addLabel("CI-${Machine}-Failed") - def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) - pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}") - if (fileExists('${HOME}/RUNTESTS/ci.log')) { - def fileContent = readFile '${HOME}/RUNTESTS/ci.log' - fileContent.eachLine { line -> - if( line.contains(".log")) { - archiveArtifacts artifacts: "${line}", fingerprint: true - } - } - } - } - } - } - } - } - } - } - } - } - -} -- GitLab From c8c48381c675533b2e71e1a1c246dd4ff67a048b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net> Date: Thu, 8 Feb 2024 14:36:23 -0500 Subject: [PATCH 447/447] updated format with one more <br> --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d7c88a96..c591aae70 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -129,7 +129,7 @@ pipeline { HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments ws(HOMEgfs) { pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() - pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory `${HOME}/RUNTESTS/${pslot}`") + pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`") try { sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) } catch (Exception e) { -- GitLab