From be21834d054adbfb4438ae5ae43f0c74fb04712b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 18:58:29 +0000
Subject: [PATCH 001/447] added basic Jenkinsfile

---
 Jenkinsfile | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..b663b5297
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,22 @@
+pipeline {
+    agent DemoJNLP
+
+    environment {
+        REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
+        PR_NUMBER = '203'
+        GITHUB_TOKEN = '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'
+    }
+
+
+    stage('Checkout') {
+       steps {
+              git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}"
+             }
+        }
+
+        stage('Build') {
+            steps {
+                sh 'touch this_examle'
+            }
+        }
+    }
-- 
GitLab


From c969c8d6b936071496b8ccff54d169fa1c25e81c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 19:13:57 +0000
Subject: [PATCH 002/447] added node name to Jenkinsfile

---
 Jenkinsfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index b663b5297..488a08c5a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,6 +7,7 @@ pipeline {
         GITHUB_TOKEN = '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'
     }
 
+    node ('DemoJNPL') {
 
     stage('Checkout') {
        steps {
@@ -20,3 +21,5 @@ pipeline {
             }
         }
     }
+
+    }
-- 
GitLab


From b94a115ca7db3ee0987ba64491b36b49aee067e3 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Thu, 7 Dec 2023 19:26:32 +0000
Subject: [PATCH 003/447] Update Jenkinsfile

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 488a08c5a..03c30339c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,6 @@
 pipeline {
-    agent DemoJNLP
+    agent('DemoJNLP') {
+  
 
     environment {
         REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
@@ -23,3 +24,4 @@ pipeline {
     }
 
     }
+}
-- 
GitLab


From 79148ec51e8d2a7c2bc22becc356d7cc11f70096 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 19:46:01 +0000
Subject: [PATCH 004/447] updaet syntax on Jenkinsfile

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 03c30339c..52b9a8651 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,5 @@
 pipeline {
-    agent('DemoJNLP') {
-  
+   agent( label 'demoJNLP')
 
     environment {
         REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
@@ -10,7 +9,9 @@ pipeline {
 
     node ('DemoJNPL') {
 
-    stage('Checkout') {
+    stages {    
+
+       stage('Checkout') {
        steps {
               git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}"
              }
@@ -23,5 +24,4 @@ pipeline {
         }
     }
 
-    }
-}
+}
\ No newline at end of file
-- 
GitLab


From 8096cee74d07b6f8f492e5432b7d5c19c28ae908 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 19:53:39 +0000
Subject: [PATCH 005/447] brackets on agaent

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 52b9a8651..7b2495925 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,5 @@
 pipeline {
-   agent( label 'demoJNLP')
+   agent{ label 'demoJNLP'}
 
     environment {
         REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
-- 
GitLab


From 5cb82cdde47350d1fb13a8a8492f14e82866a366 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 19:56:53 +0000
Subject: [PATCH 006/447] missed bracket

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7b2495925..a43481ace 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -23,5 +23,5 @@ pipeline {
             }
         }
     }
-
+   }
 }
\ No newline at end of file
-- 
GitLab


From ed3e612af55c81b080c3098133d78c09792aa975 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 20:00:22 +0000
Subject: [PATCH 007/447] node goes in stages over each step

---
 Jenkinsfile | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a43481ace..b1010c02c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,14 +7,12 @@ pipeline {
         GITHUB_TOKEN = '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'
     }
 
-    node ('DemoJNPL') {
-
     stages {    
-
+       node ('DemoJNPL') {
        stage('Checkout') {
-       steps {
+         steps {
               git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}"
-             }
+         }
         }
 
         stage('Build') {
-- 
GitLab


From 8f1baeb4eac9c60d46c71916e8e5911ed0853d51 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 7 Dec 2023 20:05:25 +0000
Subject: [PATCH 008/447] took node out for now

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b1010c02c..311792f27 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,7 +8,6 @@ pipeline {
     }
 
     stages {    
-       node ('DemoJNPL') {
        stage('Checkout') {
          steps {
               git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}"
@@ -21,5 +20,4 @@ pipeline {
             }
         }
     }
-   }
 }
\ No newline at end of file
-- 
GitLab


From 95d209d27e2cee459f0163e0f22d00aa470ac31d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 20:49:39 +0000
Subject: [PATCH 009/447] do a push

---
 ci/scripts/noop | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 ci/scripts/noop

diff --git a/ci/scripts/noop b/ci/scripts/noop
new file mode 100644
index 000000000..e69de29bb
-- 
GitLab


From bbdbf7b9daca54b614bd275856b53d7f971cf8e3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 21:14:42 +0000
Subject: [PATCH 010/447] checkout PR

---
 Jenkinsfile | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 311792f27..f3d86c7a5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,23 +1,12 @@
 pipeline {
    agent{ label 'demoJNLP'}
 
-    environment {
-        REPO_URL = 'https://github.com/TerrenceMcGuinness-NOAA/global-workflow.git'
-        PR_NUMBER = '203'
-        GITHUB_TOKEN = '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'
-    }
-
     stages {    
-       stage('Checkout') {
-         steps {
-              git branch: "refs/pull/${PR_NUMBER}/head", url: "${REPO_URL}"
-         }
-        }
-
-        stage('Build') {
+        stage('Checkout Repos') {
             steps {
-                sh 'touch this_examle'
+                sh './source/checkout.sh -c -g -u'
             }
+
         }
     }
 }
\ No newline at end of file
-- 
GitLab


From 52ae1aa72130e17cf4bb24404797809f914ca1c0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 21:22:50 +0000
Subject: [PATCH 011/447] updated path to checkout.sh

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f3d86c7a5..266d34344 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,9 +4,9 @@ pipeline {
     stages {    
         stage('Checkout Repos') {
             steps {
-                sh './source/checkout.sh -c -g -u'
+                sh './sorc/checkout.sh -c -g -u'
             }
 
         }
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From 7e043e3d326a4c5e16a31bb7877401c4a411bcd8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 21:51:32 +0000
Subject: [PATCH 012/447] add label

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 266d34344..7d4d6517a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -6,7 +6,7 @@ pipeline {
             steps {
                 sh './sorc/checkout.sh -c -g -u'
             }
-
+            githubPRAddLabels labelProperty: labels('CI-Orion-Built')
         }
     }
 }
-- 
GitLab


From 8b23d6e9bd833a53116f43464392896d2b827249 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 22:04:41 +0000
Subject: [PATCH 013/447] add label

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7d4d6517a..e2a298090 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,9 +4,9 @@ pipeline {
     stages {    
         stage('Checkout Repos') {
             steps {
-                sh './sorc/checkout.sh -c -g -u'
+               sh './sorc/checkout.sh -c -g -u'
+               pullRequest.addLabel('CI-Orion-Passed')
             }
-            githubPRAddLabels labelProperty: labels('CI-Orion-Built')
         }
     }
 }
-- 
GitLab


From 10986b48b35f4f6f78bc80dfa394d74384759e53 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 22:14:22 +0000
Subject: [PATCH 014/447] noop change

---
 ci/scripts/noop | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 ci/scripts/noop

diff --git a/ci/scripts/noop b/ci/scripts/noop
deleted file mode 100644
index e69de29bb..000000000
-- 
GitLab


From a4ccc5e3fa58b3d82a6f67f1d79fb6903e6dc6ad Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 22:16:17 +0000
Subject: [PATCH 015/447] moving add Label around

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e2a298090..28351ebe8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -5,8 +5,8 @@ pipeline {
         stage('Checkout Repos') {
             steps {
                sh './sorc/checkout.sh -c -g -u'
-               pullRequest.addLabel('CI-Orion-Passed')
             }
+            pullRequest.addLabel('CI-Orion-Passed')
         }
     }
 }
-- 
GitLab


From 907d31aa09ce5670d6fe0df291df23e525402048 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 8 Dec 2023 22:24:57 +0000
Subject: [PATCH 016/447] added labels add in post SUCCESS

---
 Jenkinsfile | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 28351ebe8..69062908f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,12 +1,18 @@
 pipeline {
-   agent{ label 'demoJNLP'}
+    agent{ label 'demoJNLP'}
 
     stages {    
         stage('Checkout Repos') {
             steps {
-               sh './sorc/checkout.sh -c -g -u'
+                sh './sorc/checkout.sh -c -g -u'
             }
-            pullRequest.addLabel('CI-Orion-Passed')
         }
     }
-}
+    post {
+        success {
+            script {
+                pullRequest.labels.add('CI-Orion-Passed')
+            }
+        }
+    }
+}
\ No newline at end of file
-- 
GitLab


From 29de307023159567512a5f556ac7320c3739c29e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 11 Dec 2023 17:30:38 +0000
Subject: [PATCH 017/447] added correct credntialsId for label add

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 69062908f..bf60bfdda 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,6 +11,7 @@ pipeline {
     post {
         success {
             script {
+                withCredentials([string(credentialsId: '7dcc5bf3-cc6a-48b1-acd8-c1dc35e3b10b', variable: 'GITHUB_TOKEN')])
                 pullRequest.labels.add('CI-Orion-Passed')
             }
         }
-- 
GitLab


From 772f8b84285cdae7f5f96f950575f8a9fb07b87b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 11 Dec 2023 18:20:59 +0000
Subject: [PATCH 018/447] removed credentials block

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bf60bfdda..69062908f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,7 +11,6 @@ pipeline {
     post {
         success {
             script {
-                withCredentials([string(credentialsId: '7dcc5bf3-cc6a-48b1-acd8-c1dc35e3b10b', variable: 'GITHUB_TOKEN')])
                 pullRequest.labels.add('CI-Orion-Passed')
             }
         }
-- 
GitLab


From b6568de7e83ee605eecf5145df8fed2c8d6b6c8a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 11 Dec 2023 21:10:07 +0000
Subject: [PATCH 019/447] updated Node name

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 69062908f..cf58cb432 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,5 @@
 pipeline {
-    agent{ label 'demoJNLP'}
+    agent{ label 'Sanbox'}
 
     stages {    
         stage('Checkout Repos') {
-- 
GitLab


From e09e1dd6caa4e3c6dbe4d49097ef8a8bee95c1f8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 14 Dec 2023 20:58:40 +0000
Subject: [PATCH 020/447] basic sm and configured to include submodules

---
 Jenkinsfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index cf58cb432..30f632841 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,11 +1,10 @@
 pipeline {
-    agent{ label 'Sanbox'}
+    agent{ label 'local'}
 
     stages {    
-        stage('Checkout Repos') {
-            steps {
-                sh './sorc/checkout.sh -c -g -u'
-            }
+        stage('Checkout') {
+        node{ 
+          checkout scm
         }
     }
     post {
@@ -15,4 +14,4 @@ pipeline {
             }
         }
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From 950ca196968429386ac60b4fad70e81bcb7b0331 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 14 Dec 2023 21:13:17 +0000
Subject: [PATCH 021/447] missed a bracket in Jenkinsfile

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 30f632841..3f055a759 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,8 +3,9 @@ pipeline {
 
     stages {    
         stage('Checkout') {
-        node{ 
-          checkout scm
+            node{ 
+                checkout scm
+            }
         }
     }
     post {
-- 
GitLab


From c23e8bda3715bf8ea7d6cb907689782861d60b9b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 14 Dec 2023 21:26:31 +0000
Subject: [PATCH 022/447] repalced node with step becuse first block in stage
 is steps

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3f055a759..89e3c239d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,7 +3,7 @@ pipeline {
 
     stages {    
         stage('Checkout') {
-            node{ 
+            steps { 
                 checkout scm
             }
         }
-- 
GitLab


From c3bf6b2ef63dc26e4543dc9744fd559ee5635b55 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 14 Dec 2023 22:17:19 +0000
Subject: [PATCH 023/447] updated methond addLabel for pullResest

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 89e3c239d..aef4f1c2e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,7 +11,7 @@ pipeline {
     post {
         success {
             script {
-                pullRequest.labels.add('CI-Orion-Passed')
+                pullRequest.addLabel(['CI-Orion-Passed'])  
             }
         }
     }
-- 
GitLab


From 79b7bdcb3486465254d024a38378a8aea104c65a Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Tue, 19 Dec 2023 13:07:54 -0600
Subject: [PATCH 024/447] added label to node to match that on EPICs controler
 orion-emc

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index aef4f1c2e..254120396 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,5 @@
 pipeline {
-    agent{ label 'local'}
+    agent{ label 'orion-emc'}
 
     stages {    
         stage('Checkout') {
-- 
GitLab


From 9380545e36125e6922b41406bcc40432374c5d82 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 20:58:39 +0000
Subject: [PATCH 025/447] added simple full checkout and FF build and the
 C48_ATM experment dir

---
 Jenkinsfile | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 254120396..7a9af735a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,11 +3,32 @@ pipeline {
 
     stages {    
         stage('Checkout') {
-            steps { 
+            steps {
                 checkout scm
+                sh 'git submodule update --init --recursive'
+            }
+        }
+        stage('Build') {
+          steps {
+            sh 'sorc/build_all.sh'
+          }
+        }
+        stage ('Create Experment') {
+            steps {
+               sh '''
+                  export HOMEgfs=$PWD
+                  mkdir -p RUNTESTS
+                  export RUNTESTS=$PWD/RUNTESTS
+                  source ci/platforms/config.orion.sh
+                  pr_sha=$(git rev-parse --short HEAD)
+                  case=C48_ATM
+                  export pslot=${case}_${pr_sha}
+                  workflow/create_experment.py --yaml ci/cases/pr/${case}'
+                  '''
             }
         }
     }
+
     post {
         success {
             script {
-- 
GitLab


From 4b41cdc2cc6a92fb2caf1cd431af01b7b6d39cb5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 21:26:33 +0000
Subject: [PATCH 026/447] added link as a build step, fixed typo with
 cnofig.orion call

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7a9af735a..bdbac19f7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,6 +11,7 @@ pipeline {
         stage('Build') {
           steps {
             sh 'sorc/build_all.sh'
+            sh 'sourc/link_workflow.sh'
           }
         }
         stage ('Create Experment') {
@@ -19,7 +20,7 @@ pipeline {
                   export HOMEgfs=$PWD
                   mkdir -p RUNTESTS
                   export RUNTESTS=$PWD/RUNTESTS
-                  source ci/platforms/config.orion.sh
+                  source ci/platforms/config.orion
                   pr_sha=$(git rev-parse --short HEAD)
                   case=C48_ATM
                   export pslot=${case}_${pr_sha}
-- 
GitLab


From 10263d59138e183964d5bdf77ccf61a4f495c1a7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 21:48:45 +0000
Subject: [PATCH 027/447] corrected pash to global link script ins sorc

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bdbac19f7..2bf356aec 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,7 +11,7 @@ pipeline {
         stage('Build') {
           steps {
             sh 'sorc/build_all.sh'
-            sh 'sourc/link_workflow.sh'
+            sh 'sorc/link_workflow.sh'
           }
         }
         stage ('Create Experment') {
-- 
GitLab


From fb3b4a192bccf31f230a399f20394d5ca37c4e4f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 22:24:49 +0000
Subject: [PATCH 028/447] hanging quote in sh for creating an experment

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2bf356aec..d770d4b94 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
                   pr_sha=$(git rev-parse --short HEAD)
                   case=C48_ATM
                   export pslot=${case}_${pr_sha}
-                  workflow/create_experment.py --yaml ci/cases/pr/${case}'
+                  workflow/create_experment.py --yaml ci/cases/pr/${case}
                   '''
             }
         }
-- 
GitLab


From 8970be928ff7cf5756729b5ebbabafa59e294cac Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 23:09:40 +0000
Subject: [PATCH 029/447] missed i in experiment in Jenkinsfile

---
 Jenkinsfile | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d770d4b94..af0ad1cbe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -5,6 +5,8 @@ pipeline {
         stage('Checkout') {
             steps {
                 checkout scm
+                pullRequest.removeLabel('CI-Orion-Ready')
+                pullRequest.addLabel('CI-Orion-Building')  
                 sh 'git submodule update --init --recursive'
             }
         }
@@ -16,16 +18,19 @@ pipeline {
         }
         stage ('Create Experment') {
             steps {
-               sh '''
-                  export HOMEgfs=$PWD
-                  mkdir -p RUNTESTS
-                  export RUNTESTS=$PWD/RUNTESTS
-                  source ci/platforms/config.orion
-                  pr_sha=$(git rev-parse --short HEAD)
-                  case=C48_ATM
-                  export pslot=${case}_${pr_sha}
-                  workflow/create_experment.py --yaml ci/cases/pr/${case}
-                  '''
+                pullRequest.removeLabel('CI-Orion-Building')
+                pullRequest.addLabel('CI-Orion-Running')  
+                sh '''
+                   export HOMEgfs=$PWD
+                   mkdir -p RUNTESTS
+                   export RUNTESTS=$PWD/RUNTESTS
+                   source ci/platforms/config.orion
+                   pr_sha=$(git rev-parse --short HEAD)
+                   case=C48_ATM
+                   export pslot=${case}_${pr_sha}
+                   workflow/create_experiment.py --yaml ci/cases/pr/${case}
+                   '''
+               pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
             }
         }
     }
@@ -33,7 +38,8 @@ pipeline {
     post {
         success {
             script {
-                pullRequest.addLabel(['CI-Orion-Passed'])  
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Passed')  
             }
         }
     }
-- 
GitLab


From 14b722e77346b366895c62894bdace3dd894aeee Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 23:29:59 +0000
Subject: [PATCH 030/447] added scripts around pullRequest methods in
 Jenkinsfile

---
 Jenkinsfile | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index af0ad1cbe..0e5e73511 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -5,8 +5,10 @@ pipeline {
         stage('Checkout') {
             steps {
                 checkout scm
-                pullRequest.removeLabel('CI-Orion-Ready')
-                pullRequest.addLabel('CI-Orion-Building')  
+                script {
+                   pullRequest.removeLabel('CI-Orion-Ready')
+                   pullRequest.addLabel('CI-Orion-Building')
+                }
                 sh 'git submodule update --init --recursive'
             }
         }
@@ -18,8 +20,10 @@ pipeline {
         }
         stage ('Create Experment') {
             steps {
-                pullRequest.removeLabel('CI-Orion-Building')
-                pullRequest.addLabel('CI-Orion-Running')  
+                script {
+                  pullRequest.removeLabel('CI-Orion-Building')
+                  pullRequest.addLabel('CI-Orion-Running')  
+                }
                 sh '''
                    export HOMEgfs=$PWD
                    mkdir -p RUNTESTS
@@ -28,9 +32,11 @@ pipeline {
                    pr_sha=$(git rev-parse --short HEAD)
                    case=C48_ATM
                    export pslot=${case}_${pr_sha}
-                   workflow/create_experiment.py --yaml ci/cases/pr/${case}
+                   workflow/create_experment.py --yaml ci/cases/pr/${case}
                    '''
-               pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
+                script {   
+                  pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
+                }
             }
         }
     }
@@ -42,5 +48,11 @@ pipeline {
                 pullRequest.addLabel('CI-Orion-Passed')  
             }
         }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Failed')  
+            }
+        }
     }
 }
-- 
GitLab


From 4564388f522ba529835175d3a83d78291022b987 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 20 Dec 2023 23:55:02 +0000
Subject: [PATCH 031/447] missed update to experiment in create experiment
 script in Jenkins file (another redu)

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0e5e73511..9694723de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,7 +32,7 @@ pipeline {
                    pr_sha=$(git rev-parse --short HEAD)
                    case=C48_ATM
                    export pslot=${case}_${pr_sha}
-                   workflow/create_experment.py --yaml ci/cases/pr/${case}
+                   workflow/create_experiment.py --yaml ci/cases/pr/${case}
                    '''
                 script {   
                   pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
-- 
GitLab


From 03de881532776321efd0b53bae376bded841ce50 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 20 Dec 2023 20:33:23 -0500
Subject: [PATCH 032/447] Update Jenkinsfile

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9694723de..67d21dfcc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,6 +32,7 @@ pipeline {
                    pr_sha=$(git rev-parse --short HEAD)
                    case=C48_ATM
                    export pslot=${case}_${pr_sha}
+                   source workflow/gwsetup.sh
                    workflow/create_experiment.py --yaml ci/cases/pr/${case}
                    '''
                 script {   
-- 
GitLab


From 6849a1646890d2d2fbbedc53db5a277636d9cd13 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 21 Dec 2023 01:55:54 +0000
Subject: [PATCH 033/447] misspelled another path ... underscore in gw setup

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 67d21dfcc..128738ab0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,7 +32,7 @@ pipeline {
                    pr_sha=$(git rev-parse --short HEAD)
                    case=C48_ATM
                    export pslot=${case}_${pr_sha}
-                   source workflow/gwsetup.sh
+                   source workflow/gw_setup.sh
                    workflow/create_experiment.py --yaml ci/cases/pr/${case}
                    '''
                 script {   
-- 
GitLab


From bf49b89ef996494a41d9d87e00ff6f90b8946d33 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 21 Dec 2023 02:20:58 +0000
Subject: [PATCH 034/447] need extention yaml on case name

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 128738ab0..1b2b33e4c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -33,7 +33,7 @@ pipeline {
                    case=C48_ATM
                    export pslot=${case}_${pr_sha}
                    source workflow/gw_setup.sh
-                   workflow/create_experiment.py --yaml ci/cases/pr/${case}
+                   workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
                    '''
                 script {   
                   pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
-- 
GitLab


From b4792992fbceb399356d42d1a0ac9295dff42088 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 16:42:40 +0000
Subject: [PATCH 035/447] renaming Jenkinsfile inorder to remove passed run
 logs on controler

---
 Jenkinsfile => Jenkinsfile_temp | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile => Jenkinsfile_temp (100%)

diff --git a/Jenkinsfile b/Jenkinsfile_temp
similarity index 100%
rename from Jenkinsfile
rename to Jenkinsfile_temp
-- 
GitLab


From 38c51cdc238ab502d5c028f519049092d4742d8f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 16:44:21 +0000
Subject: [PATCH 036/447] rename Jenkinsfile back to org to start fresh pipline

---
 Jenkinsfile_temp => Jenkinsfile | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile_temp => Jenkinsfile (100%)

diff --git a/Jenkinsfile_temp b/Jenkinsfile
similarity index 100%
rename from Jenkinsfile_temp
rename to Jenkinsfile
-- 
GitLab


From 9557169820d2391c0f3c567d322a22624559715d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 19:01:37 +0000
Subject: [PATCH 037/447] generlized stages for creating each pr case in
 pipeline

---
 Jenkinsfile                          | 46 ++++++++++++++++------------
 ci/scripts/utils/ci_utils.sh         | 13 ++++++++
 ci/scripts/utils/ci_utils_wrapper.sh |  6 ++++
 3 files changed, 46 insertions(+), 19 deletions(-)
 create mode 100755 ci/scripts/utils/ci_utils_wrapper.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index 1b2b33e4c..52f10d1de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,11 @@
 pipeline {
     agent{ label 'orion-emc'}
 
+    environment {
+        HOMEgfs = "${WORKSPACE}"
+        RUNTESTS = "${WORKSPACE}/RUNTESTS"
+    }
+
     stages {    
         stage('Checkout') {
             steps {
@@ -16,30 +21,33 @@ pipeline {
           steps {
             sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
+            env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
           }
         }
-        stage ('Create Experment') {
-            steps {
-                script {
-                  pullRequest.removeLabel('CI-Orion-Building')
-                  pullRequest.addLabel('CI-Orion-Running')  
-                }
-                sh '''
-                   export HOMEgfs=$PWD
-                   mkdir -p RUNTESTS
-                   export RUNTESTS=$PWD/RUNTESTS
-                   source ci/platforms/config.orion
-                   pr_sha=$(git rev-parse --short HEAD)
-                   case=C48_ATM
-                   export pslot=${case}_${pr_sha}
-                   source workflow/gw_setup.sh
-                   workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
-                   '''
-                script {   
-                  pullRequest.comment('SUCCESS creating experment C48_ATM on Orion') 
+
+        def cases = env.CASE_LIST.tokenize('\n')
+
+        cases.each { case_name ->
+            stage("Run ${case_name}") {
+                steps {
+                    sh '''
+                       export HOMEgfs=${env.HOMEgfs}
+                       mkdir -p ${env.RUNTESTS}
+                       export RUNTESTS=${env.RUNTESTS}
+                       source ci/platforms/config.orion
+                       pr_sha=$(git rev-parse --short HEAD)
+                       case=${case_name}
+                       export pslot=${case}_${pr_sha}
+                       source workflow/gw_setup.sh
+                       workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
+                       '''
+                    script {
+                        pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+                    }
                 }
             }
         }
+
     }
 
     post {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 737a3e5a8..cc9f8aaa2 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -22,3 +22,16 @@ function cancel_slurm_jobs() {
     fi
   done
 }
+
+function get_pr_case_list () {
+
+    #############################################################
+    # loop over every yaml file in the PR's ci/cases
+    # and create an run directory for each one for this PR loop
+    #############################################################
+    echo ${HOMEgfs}
+    for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do
+      case=$(basename "${yaml_config}" .yaml) || true
+      echo "${case}"
+    done
+}
\ No newline at end of file
diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh
new file mode 100755
index 000000000..f3cd461e8
--- /dev/null
+++ b/ci/scripts/utils/ci_utils_wrapper.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+utitilty_function="${1}"
+
+source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
+${utitilty_function} "${@:2}"
\ No newline at end of file
-- 
GitLab


From d959942da3fb60935f204d9227983be5d8d90843 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 19:10:59 +0000
Subject: [PATCH 038/447] Jenkinsfile syntax step over env def

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 52f10d1de..74ddceb56 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -21,6 +21,8 @@ pipeline {
           steps {
             sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
+          }
+          steps {
             env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
           }
         }
-- 
GitLab


From afb3789f412569218928b331a66d1b4fbb54a392 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 19:46:36 +0000
Subject: [PATCH 039/447] simply made cases pipeline global

---
 Jenkinsfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 74ddceb56..cf541e973 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,3 +1,5 @@
+
+def cases="cases"
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -21,14 +23,13 @@ pipeline {
           steps {
             sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
-          }
-          steps {
-            env.CASE_LIST = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+            script {
+              case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              cases=case_list.tokenize('\n')
+            }
           }
         }
 
-        def cases = env.CASE_LIST.tokenize('\n')
-
         cases.each { case_name ->
             stage("Run ${case_name}") {
                 steps {
-- 
GitLab


From b7a7a6042714b6256095705a4c1397643e0093f2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 19:57:57 +0000
Subject: [PATCH 040/447] adding next of cases as a new stage

---
 Jenkinsfile | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index cf541e973..9a3d44b11 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -29,9 +29,11 @@ pipeline {
             }
           }
         }
-
-        cases.each { case_name ->
-            stage("Run ${case_name}") {
+ 
+        stage('Create Cases') {
+          stages {
+            cases.each { case_name ->
+                stage("Run ${case_name}") {
                 steps {
                     sh '''
                        export HOMEgfs=${env.HOMEgfs}
@@ -48,9 +50,10 @@ pipeline {
                         pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                     }
                 }
+
             }
+          }
         }
-
     }
 
     post {
-- 
GitLab


From 98068be1a084163aeea3c87c2e8b09351346a7ec Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 20:02:05 +0000
Subject: [PATCH 041/447] missed a bracket in new nest of cases

---
 Jenkinsfile | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9a3d44b11..e9954624e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -31,28 +31,29 @@ pipeline {
         }
  
         stage('Create Cases') {
-          stages {
-            cases.each { case_name ->
-                stage("Run ${case_name}") {
-                steps {
-                    sh '''
-                       export HOMEgfs=${env.HOMEgfs}
-                       mkdir -p ${env.RUNTESTS}
-                       export RUNTESTS=${env.RUNTESTS}
-                       source ci/platforms/config.orion
-                       pr_sha=$(git rev-parse --short HEAD)
-                       case=${case_name}
-                       export pslot=${case}_${pr_sha}
-                       source workflow/gw_setup.sh
-                       workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
-                       '''
-                    script {
-                        pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+            stages {
+                cases.each { case_name ->
+                    stage("Run ${case_name}") {
+                        steps {
+                          sh '''
+                          export HOMEgfs=${env.HOMEgfs}
+                          mkdir -p ${env.RUNTESTS}
+                          export RUNTESTS=${env.RUNTESTS}
+                          source ci/platforms/config.orion
+                          pr_sha=$(git rev-parse --short HEAD)
+                          case=${case_name}
+                          export pslot=${case}_${pr_sha}
+                          source workflow/gw_setup.sh
+                          workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
+                          '''
+                          script {
+                            pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+                          }
+                        }
+
                     }
                 }
-
             }
-          }
         }
     }
 
-- 
GitLab


From 6b478014d3eda26aabffc047198cc469e771ddef Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 20:22:34 +0000
Subject: [PATCH 042/447] dynamic stages as steps script

---
 Jenkinsfile | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e9954624e..9fb0b6f4f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -31,30 +31,32 @@ pipeline {
         }
  
         stage('Create Cases') {
-            stages {
-                cases.each { case_name ->
-                    stage("Run ${case_name}") {
-                        steps {
-                          sh '''
-                          export HOMEgfs=${env.HOMEgfs}
-                          mkdir -p ${env.RUNTESTS}
-                          export RUNTESTS=${env.RUNTESTS}
-                          source ci/platforms/config.orion
-                          pr_sha=$(git rev-parse --short HEAD)
-                          case=${case_name}
-                          export pslot=${case}_${pr_sha}
-                          source workflow/gw_setup.sh
-                          workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
-                          '''
-                          script {
-                            pullRequest.comment("SUCCESS creating ${case_name} on Orion")
-                          }
+            steps {
+                script {
+                    cases.each { case_name ->
+                        stage("Run ${case_name}") {
+                            steps {
+                              sh '''
+                              export HOMEgfs=${env.HOMEgfs}
+                              mkdir -p ${env.RUNTESTS}
+                              export RUNTESTS=${env.RUNTESTS}
+                              source ci/platforms/config.orion
+                              pr_sha=$(git rev-parse --short HEAD)
+                              case=${case_name}
+                              export pslot=${case}_${pr_sha}
+                              source workflow/gw_setup.sh
+                              workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
+                              '''
+                              script {
+                              pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+                              }
+                            }
                         }
-
                     }
                 }
             }
-        }
+        }    
+
     }
 
     post {
-- 
GitLab


From 547067ce76394bc8c3f6c6b220d5d71528ede5a0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 20:33:21 +0000
Subject: [PATCH 043/447] rename Jenkinsfile to clear

---
 Jenkinsfile => Jenkinsfile_org | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile => Jenkinsfile_org (100%)

diff --git a/Jenkinsfile b/Jenkinsfile_org
similarity index 100%
rename from Jenkinsfile
rename to Jenkinsfile_org
-- 
GitLab


From 3624df74ac56d3bdcb289025264aa7cfc65b408d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 20:35:34 +0000
Subject: [PATCH 044/447] rename Jenkinsfile to clear

---
 Jenkinsfile_org => Jenkinsfile | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile_org => Jenkinsfile (100%)

diff --git a/Jenkinsfile_org b/Jenkinsfile
similarity index 100%
rename from Jenkinsfile_org
rename to Jenkinsfile
-- 
GitLab


From 750b67f517603b384ca9b0aa6a8b3ba151a270e4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:16:53 +0000
Subject: [PATCH 045/447] check case liset

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9fb0b6f4f..87720432c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -21,12 +21,13 @@ pipeline {
         }
         stage('Build') {
           steps {
-            sh 'sorc/build_all.sh'
+            // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
               case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
               cases=case_list.tokenize('\n')
             }
+            echo "cases: ${cases}"
           }
         }
  
-- 
GitLab


From 5edb4b17cb11a774b73b5958630c8e0beebd7360 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:18:31 +0000
Subject: [PATCH 046/447] check case liset

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 87720432c..3406f560e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -34,6 +34,8 @@ pipeline {
         stage('Create Cases') {
             steps {
                 script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')  
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
                             steps {
-- 
GitLab


From 04bf11e4f1c3c07b6bb89aaa924f2e0b1d55fedc Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:26:20 +0000
Subject: [PATCH 047/447] check case list in next stage and remove first
 element

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3406f560e..d35123590 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,7 +35,9 @@ pipeline {
             steps {
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')  
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case.removeAt(0)
+                    echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
                             steps {
-- 
GitLab


From e7e3a7e31fe98b6cf626b9685094fb032cde9e01 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:28:44 +0000
Subject: [PATCH 048/447] mispelled cases as case

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d35123590..d29c77415 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -36,7 +36,7 @@ pipeline {
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
                     pullRequest.addLabel('CI-Orion-Running')
-                    case.removeAt(0)
+                    cases.removeAt(0)
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
-- 
GitLab


From b00a597d1574bea543e61166efcbc2d622eb51e1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:35:41 +0000
Subject: [PATCH 049/447] remove first element before making global for cases
 list

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d29c77415..10eaa8cff 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -25,6 +25,7 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              case_list.removeAt(0)
               cases=case_list.tokenize('\n')
             }
             echo "cases: ${cases}"
@@ -36,7 +37,6 @@ pipeline {
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
                     pullRequest.addLabel('CI-Orion-Running')
-                    cases.removeAt(0)
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
-- 
GitLab


From a914b85cf30449ca0ffdd3c8042528fc18d4ff9d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:41:16 +0000
Subject: [PATCH 050/447] remove firs element locally after tokenize case list

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 10eaa8cff..fe0ea6966 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -25,10 +25,11 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              case_list.tokenize('\n')
               case_list.removeAt(0)
-              cases=case_list.tokenize('\n')
+              cases=case_list
+              echo "cases: ${cases}"
             }
-            echo "cases: ${cases}"
           }
         }
  
-- 
GitLab


From e1f33f76676320266b6e579066e5c6bfa5c0c460 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:44:51 +0000
Subject: [PATCH 051/447] create seperate ouput string then tokenized it before
 removing first elmenent

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe0ea6966..3fc0dcf0b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -24,8 +24,8 @@ pipeline {
             // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
-              case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              case_list.tokenize('\n')
+              case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              case_list = case_list_ouput.tokenize('\n')
               case_list.removeAt(0)
               cases=case_list
               echo "cases: ${cases}"
-- 
GitLab


From 489f198a007eef4ab6b016988c570ccce30df860 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:50:29 +0000
Subject: [PATCH 052/447] workaround admin permissions to use removeAt

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3fc0dcf0b..336f3fcf6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,6 @@
 
 def cases="cases"
+def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -24,9 +25,9 @@ pipeline {
             // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
-              case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
               case_list = case_list_ouput.tokenize('\n')
-              case_list.removeAt(0)
+              //case_list.removeAt(0)
               cases=case_list
               echo "cases: ${cases}"
             }
-- 
GitLab


From 3f7312a5ad549cff5c4e1bf419812a1588f24f17 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:51:02 +0000
Subject: [PATCH 053/447] workaround admin permissions to use removeAt

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 336f3fcf6..1811703a6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -28,6 +28,7 @@ pipeline {
               //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
               case_list = case_list_ouput.tokenize('\n')
               //case_list.removeAt(0)
+              case_list.remove(0)
               cases=case_list
               echo "cases: ${cases}"
             }
-- 
GitLab


From c6d508c332c9501bd1224e8bfe10031689530bf8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:54:24 +0000
Subject: [PATCH 054/447] misspelled case list outout

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1811703a6..73c45d401 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,7 +26,7 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              case_list = case_list_ouput.tokenize('\n')
+              case_list = case_list_output.tokenize('\n')
               //case_list.removeAt(0)
               case_list.remove(0)
               cases=case_list
-- 
GitLab


From be720eb82fea3038a45e0638d5df71bc26160d2f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 21:56:07 +0000
Subject: [PATCH 055/447] spaces on workaround

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 73c45d401..32e6e5c8b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,7 +26,7 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              case_list = case_list_output.tokenize('\n')
+              case_list = case_list_output.tokenize(' ')
               //case_list.removeAt(0)
               case_list.remove(0)
               cases=case_list
-- 
GitLab


From 8925066b33b8f22ab2c5e0fd96136fa1d1ffacd7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:10:00 +0000
Subject: [PATCH 056/447] took out steps in create cases stage because its
 scripited

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 32e6e5c8b..6816797bb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -43,7 +43,6 @@ pipeline {
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
-                            steps {
                               sh '''
                               export HOMEgfs=${env.HOMEgfs}
                               mkdir -p ${env.RUNTESTS}
@@ -58,7 +57,6 @@ pipeline {
                               script {
                               pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                               }
-                            }
                         }
                     }
                 }
-- 
GitLab


From 8abc5d058ec6ce22021ec6b5f5bb5273e4f7e7f9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:10:47 +0000
Subject: [PATCH 057/447] took out steps in create cases stage because its
 scripited

---
 Jenkinsfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6816797bb..39f654872 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,6 @@
 
 def cases="cases"
-def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
+//def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -25,9 +25,8 @@ pipeline {
             // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
-              //case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
               case_list = case_list_output.tokenize(' ')
-              //case_list.removeAt(0)
               case_list.remove(0)
               cases=case_list
               echo "cases: ${cases}"
-- 
GitLab


From e932f4fcbbc5458d7b9044b36417d9c8c24df6bb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:13:57 +0000
Subject: [PATCH 058/447] more added back get list with remove

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 39f654872..bed73123e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -25,8 +25,8 @@ pipeline {
             // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
-              case_list_ouput = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              case_list = case_list_output.tokenize(' ')
+              case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+              case_list = case_list_output.tokenize('\n')
               case_list.remove(0)
               cases=case_list
               echo "cases: ${cases}"
-- 
GitLab


From c39c6fa33a3199b1f9f59084a6d0f90fb444ba6d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:26:10 +0000
Subject: [PATCH 059/447] same agent accross all steps

---
 Jenkinsfile | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bed73123e..6ff644109 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,7 +2,7 @@
 def cases="cases"
 //def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
 pipeline {
-    agent{ label 'orion-emc'}
+    agent none
 
     environment {
         HOMEgfs = "${WORKSPACE}"
@@ -11,6 +11,7 @@ pipeline {
 
     stages {    
         stage('Checkout') {
+        agent{ label 'orion-emc'}
             steps {
                 checkout scm
                 script {
@@ -21,6 +22,7 @@ pipeline {
             }
         }
         stage('Build') {
+        agent{ label 'orion-emc'}
           steps {
             // sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
@@ -35,6 +37,7 @@ pipeline {
         }
  
         stage('Create Cases') {
+        agent{ label 'orion-emc'}
             steps {
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
@@ -42,6 +45,8 @@ pipeline {
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
+                        agent{ label 'orion-emc'}
+                            steps {
                               sh '''
                               export HOMEgfs=${env.HOMEgfs}
                               mkdir -p ${env.RUNTESTS}
@@ -56,6 +61,7 @@ pipeline {
                               script {
                               pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                               }
+                            }
                         }
                     }
                 }
@@ -65,6 +71,7 @@ pipeline {
     }
 
     post {
+    agent{ label 'orion-emc'}
         success {
             script {
                 pullRequest.removeLabel('CI-Orion-Running')
@@ -78,4 +85,4 @@ pipeline {
             }
         }
     }
-}
+}
\ No newline at end of file
-- 
GitLab


From b74fd5856bd418436ff14f0c45aa0e3d45fc4b67 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:28:08 +0000
Subject: [PATCH 060/447] no agent in post

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6ff644109..47854dc45 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -71,7 +71,6 @@ pipeline {
     }
 
     post {
-    agent{ label 'orion-emc'}
         success {
             script {
                 pullRequest.removeLabel('CI-Orion-Running')
-- 
GitLab


From a29054115e3feca9c8dceb078648e1d9bd1d8f34 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:33:15 +0000
Subject: [PATCH 061/447] global parent stage on same agent

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 47854dc45..415689b3a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,7 +2,7 @@
 def cases="cases"
 //def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
 pipeline {
-    agent none
+    agent{ label 'orion-emc'}
 
     environment {
         HOMEgfs = "${WORKSPACE}"
-- 
GitLab


From 0a582c3e0a092dc68c25de45e66e5a88d9957eaa Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 22:43:08 +0000
Subject: [PATCH 062/447] take out steps in inner staged loop

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 415689b3a..7d9c3737f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,7 +46,7 @@ pipeline {
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
                         agent{ label 'orion-emc'}
-                            steps {
+                            //steps {
                               sh '''
                               export HOMEgfs=${env.HOMEgfs}
                               mkdir -p ${env.RUNTESTS}
@@ -61,7 +61,7 @@ pipeline {
                               script {
                               pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                               }
-                            }
+                            //}
                         }
                     }
                 }
-- 
GitLab


From f790d0feb37bb7c08989b3cea1c439fadc7f7795 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 23:13:50 +0000
Subject: [PATCH 063/447] HOMEgfs may already be assigined

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7d9c3737f..a19cdbd43 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,9 +48,7 @@ pipeline {
                         agent{ label 'orion-emc'}
                             //steps {
                               sh '''
-                              export HOMEgfs=${env.HOMEgfs}
                               mkdir -p ${env.RUNTESTS}
-                              export RUNTESTS=${env.RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
                               case=${case_name}
-- 
GitLab


From fe57fcc1eb69dea9083d0231f796a2e3616f2e0e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 23:20:47 +0000
Subject: [PATCH 064/447] still had env for RUNTESTS, removed it too

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a19cdbd43..b5407fd2a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,5 @@
 
 def cases="cases"
-//def case_list_output="C48_S2SW C48_S2SWA_gefs C48_ATM"
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -48,7 +47,7 @@ pipeline {
                         agent{ label 'orion-emc'}
                             //steps {
                               sh '''
-                              mkdir -p ${env.RUNTESTS}
+                              mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
                               case=${case_name}
-- 
GitLab


From 8f0b5f6fb6f34611485d94ca37d0f3f10e3c1c8a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 23:43:59 +0000
Subject: [PATCH 065/447] removed case and just used case_name

---
 Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b5407fd2a..5e8ee45cd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,13 +50,12 @@ pipeline {
                               mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
-                              case=${case_name}
-                              export pslot=${case}_${pr_sha}
+                              export pslot=${case_name}_${pr_sha}
                               source workflow/gw_setup.sh
-                              workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
+                              workflow/create_experiment.py --yaml ci/cases/pr/${case_name}.yaml
                               '''
                               script {
-                              pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+                               pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                               }
                             //}
                         }
-- 
GitLab


From 59c4cecd6d35263f51f2b294d5f3fbbb666402e9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 23:49:55 +0000
Subject: [PATCH 066/447] take out agent in sub stage for Run case

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5e8ee45cd..ecb49e272 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,7 +44,6 @@ pipeline {
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
-                        agent{ label 'orion-emc'}
                             //steps {
                               sh '''
                               mkdir -p ${RUNTESTS}
-- 
GitLab


From aa66559567f671b59751dbdd4de011dd4da7152e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 27 Dec 2023 23:54:00 +0000
Subject: [PATCH 067/447] case name is not in sub stage env

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ecb49e272..18b243144 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,12 +46,13 @@ pipeline {
                         stage("Run ${case_name}") {
                             //steps {
                               sh '''
+                              case=${env.case_name}
                               mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
-                              export pslot=${case_name}_${pr_sha}
+                              export pslot=${case}_${pr_sha}
                               source workflow/gw_setup.sh
-                              workflow/create_experiment.py --yaml ci/cases/pr/${case_name}.yaml
+                              workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
                               '''
                               script {
                                pullRequest.comment("SUCCESS creating ${case_name} on Orion")
-- 
GitLab


From 2347e37c8aa682b5bdf7ef30f38373e716122723 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 00:02:59 +0000
Subject: [PATCH 068/447] still can not get case name in shell

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 18b243144..91e555d29 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,9 +44,10 @@ pipeline {
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Run ${case_name}") {
-                            //steps {
+                            script {
+                               env.case = case_name
+                             }
                               sh '''
-                              case=${env.case_name}
                               mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
@@ -57,7 +58,6 @@ pipeline {
                               script {
                                pullRequest.comment("SUCCESS creating ${case_name} on Orion")
                               }
-                            //}
                         }
                     }
                 }
-- 
GitLab


From 364f2d36e6ef9d1644754ee2ab02de3127ac2c92 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 00:14:46 +0000
Subject: [PATCH 069/447] HOMEgfs gets NONE in xml so unset and local set

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 91e555d29..fe8f650be 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -53,6 +53,8 @@ pipeline {
                               pr_sha=$(git rev-parse --short HEAD)
                               export pslot=${case}_${pr_sha}
                               source workflow/gw_setup.sh
+                              unset HOMEgfs
+                              HOMEgfs=${WORKSPACE}
                               workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
                               '''
                               script {
-- 
GitLab


From 8722098e22c7ce7bd917009467a1b312500d6e4b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 00:22:11 +0000
Subject: [PATCH 070/447] remove case dirs

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe8f650be..b2367a65f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,6 +48,7 @@ pipeline {
                                env.case = case_name
                              }
                               sh '''
+                              rm -rf ${RUNTESTS}
                               mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
-- 
GitLab


From f6bb57e3e870460598d4e7aba72846e1a6e83645 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 00:32:08 +0000
Subject: [PATCH 071/447] rename Jenkinsfile to clear controler logs

---
 Jenkinsfile => Jenkinsfile_org | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile => Jenkinsfile_org (100%)

diff --git a/Jenkinsfile b/Jenkinsfile_org
similarity index 100%
rename from Jenkinsfile
rename to Jenkinsfile_org
-- 
GitLab


From b993492be1649bc7536344b00c8cfd2e3c94e37d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 00:34:33 +0000
Subject: [PATCH 072/447] rename stage Create Experiments

---
 Jenkinsfile_org | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile_org b/Jenkinsfile_org
index b2367a65f..6bdf44dfa 100644
--- a/Jenkinsfile_org
+++ b/Jenkinsfile_org
@@ -23,7 +23,7 @@ pipeline {
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
-            // sh 'sorc/build_all.sh'
+            sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
               case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
@@ -43,7 +43,7 @@ pipeline {
                     pullRequest.addLabel('CI-Orion-Running')
                     echo "cases: ${cases}"
                     cases.each { case_name ->
-                        stage("Run ${case_name}") {
+                        stage("Create ${case_name}") {
                             script {
                                env.case = case_name
                              }
-- 
GitLab


From e93e0ae02d27f544381d8ef749e5365b60b7ff4d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 27 Dec 2023 21:21:23 -0500
Subject: [PATCH 073/447] Update Jenkinsfile_org

moved RUNTESTS dir management out of loop
---
 Jenkinsfile_org | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/Jenkinsfile_org b/Jenkinsfile_org
index 6bdf44dfa..93d168820 100644
--- a/Jenkinsfile_org
+++ b/Jenkinsfile_org
@@ -4,11 +4,11 @@ pipeline {
     agent{ label 'orion-emc'}
 
     environment {
-        HOMEgfs = "${WORKSPACE}"
         RUNTESTS = "${WORKSPACE}/RUNTESTS"
     }
 
-    stages {    
+    stages {
+
         stage('Checkout') {
         agent{ label 'orion-emc'}
             steps {
@@ -20,6 +20,7 @@ pipeline {
                 sh 'git submodule update --init --recursive'
             }
         }
+
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
@@ -27,9 +28,7 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              case_list = case_list_output.tokenize('\n')
-              case_list.remove(0)
-              cases=case_list
+              cases = case_list_output.tokenize('\n').remove(0)
               echo "cases: ${cases}"
             }
           }
@@ -38,23 +37,25 @@ pipeline {
         stage('Create Cases') {
         agent{ label 'orion-emc'}
             steps {
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
+                    sh '''
+                       rm -rf ${RUNTESTS}
+                       mkdir -p ${RUNTESTS}
+                       '''
+                    script {
+                      pullRequest.removeLabel('CI-Orion-Building')
+                      pullRequest.addLabel('CI-Orion-Running')
+                    }
                     echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                             script {
                                env.case = case_name
-                             }
+                              }
                               sh '''
-                              rm -rf ${RUNTESTS}
-                              mkdir -p ${RUNTESTS}
                               source ci/platforms/config.orion
                               pr_sha=$(git rev-parse --short HEAD)
                               export pslot=${case}_${pr_sha}
                               source workflow/gw_setup.sh
-                              unset HOMEgfs
                               HOMEgfs=${WORKSPACE}
                               workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
                               '''
@@ -83,4 +84,4 @@ pipeline {
             }
         }
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From 6156b342eb11465bc82970ebb4e08e66664c6ce2 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 27 Dec 2023 21:24:11 -0500
Subject: [PATCH 074/447] Update ci_utils_wrapper.sh

added HOMEgfs defined inside wrapper
---
 ci/scripts/utils/ci_utils_wrapper.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh
index f3cd461e8..f7cb0fb89 100755
--- a/ci/scripts/utils/ci_utils_wrapper.sh
+++ b/ci/scripts/utils/ci_utils_wrapper.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+
 utitilty_function="${1}"
 
 source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
-${utitilty_function} "${@:2}"
\ No newline at end of file
+${utitilty_function} "${@:2}"
-- 
GitLab


From 9dec9fdcecadd07e4cf619423b8237171fd6856b Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 27 Dec 2023 21:27:02 -0500
Subject: [PATCH 075/447] Update Jenkinsfile_org

indents
---
 Jenkinsfile_org | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile_org b/Jenkinsfile_org
index 93d168820..fccde67a9 100644
--- a/Jenkinsfile_org
+++ b/Jenkinsfile_org
@@ -38,8 +38,8 @@ pipeline {
         agent{ label 'orion-emc'}
             steps {
                     sh '''
-                       rm -rf ${RUNTESTS}
-                       mkdir -p ${RUNTESTS}
+                        rm -rf ${RUNTESTS}
+                        mkdir -p ${RUNTESTS}
                        '''
                     script {
                       pullRequest.removeLabel('CI-Orion-Building')
@@ -52,12 +52,12 @@ pipeline {
                                env.case = case_name
                               }
                               sh '''
-                              source ci/platforms/config.orion
-                              pr_sha=$(git rev-parse --short HEAD)
-                              export pslot=${case}_${pr_sha}
-                              source workflow/gw_setup.sh
-                              HOMEgfs=${WORKSPACE}
-                              workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
+                                source ci/platforms/config.orion
+                                pr_sha=$(git rev-parse --short HEAD)
+                                export pslot=${case}_${pr_sha}
+                                source workflow/gw_setup.sh
+                                HOMEgfs=${WORKSPACE}
+                                workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
                               '''
                               script {
                                pullRequest.comment("SUCCESS creating ${case_name} on Orion")
-- 
GitLab


From d48e80042c7de1af606ab24043a1a981d085624b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:38:02 +0000
Subject: [PATCH 076/447] Moving Jenkinsfile back in

---
 Jenkinsfile_org => Jenkinsfile | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile_org => Jenkinsfile (100%)

diff --git a/Jenkinsfile_org b/Jenkinsfile
similarity index 100%
rename from Jenkinsfile_org
rename to Jenkinsfile
-- 
GitLab


From 6771e43932b22d2a147eb01d35a201e98a21af37 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:39:33 +0000
Subject: [PATCH 077/447] extra bracket at end removed

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fccde67a9..c09fef80a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -83,5 +83,4 @@ pipeline {
                 pullRequest.addLabel('CI-Orion-Failed')  
             }
         }
-    }
-}
+    }
\ No newline at end of file
-- 
GitLab


From 6e5d9723c34a6d12fb4df2717b2244fd86ae8214 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:42:32 +0000
Subject: [PATCH 078/447] echo of cases needs to be in script block

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c09fef80a..0ffe87886 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,8 +44,8 @@ pipeline {
                     script {
                       pullRequest.removeLabel('CI-Orion-Building')
                       pullRequest.addLabel('CI-Orion-Running')
+                      echo "cases: ${cases}"
                     }
-                    echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                             script {
-- 
GitLab


From 21af3995651c6988f6fbe3d79f7537395ac44433 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:49:45 +0000
Subject: [PATCH 079/447] put case each back into script block

---
 Jenkinsfile | 43 ++++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0ffe87886..e87a90bc4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -37,38 +37,35 @@ pipeline {
         stage('Create Cases') {
         agent{ label 'orion-emc'}
             steps {
-                    sh '''
-                        rm -rf ${RUNTESTS}
-                        mkdir -p ${RUNTESTS}
-                       '''
-                    script {
-                      pullRequest.removeLabel('CI-Orion-Building')
-                      pullRequest.addLabel('CI-Orion-Running')
-                      echo "cases: ${cases}"
-                    }
+                sh '''
+                   rm -rf ${RUNTESTS}
+                   mkdir -p ${RUNTESTS}
+                   '''
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                             script {
-                               env.case = case_name
-                              }
-                              sh '''
-                                source ci/platforms/config.orion
-                                pr_sha=$(git rev-parse --short HEAD)
-                                export pslot=${case}_${pr_sha}
-                                source workflow/gw_setup.sh
+                                env.case = case_name
+                            }
+                            sh '''
+                               source ci/platforms/config.orion
+                               pr_sha=$(git rev-parse --short HEAD)
+                               export pslot=${case}_${pr_sha}
+                               source workflow/gw_setup.sh
                                 HOMEgfs=${WORKSPACE}
                                 workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
-                              '''
-                              script {
-                               pullRequest.comment("SUCCESS creating ${case_name} on Orion")
-                              }
+                                '''
+                            script {
+                                  pullRequest.comment("SUCCESS creating ${case_name} on Orion")
+                            }
                         }
                     }
                 }
             }
-        }    
-
-    }
+        }
 
     post {
         success {
-- 
GitLab


From b62e71c56734d9cf60eaac4927fa034720d4e41d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:51:54 +0000
Subject: [PATCH 080/447] put missing bracket back over stages

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index e87a90bc4..70e02b781 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,6 +66,7 @@ pipeline {
                 }
             }
         }
+    }    
 
     post {
         success {
-- 
GitLab


From 9041ac42b4324d8fda3f52d46bbd33d1b738fbfa Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:53:48 +0000
Subject: [PATCH 081/447] last bracket over pipeline missing

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 70e02b781..3c31ee09c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -81,4 +81,5 @@ pipeline {
                 pullRequest.addLabel('CI-Orion-Failed')  
             }
         }
-    }
\ No newline at end of file
+    }
+}
\ No newline at end of file
-- 
GitLab


From 2b056352d1fbb58122ba1c7acce80c1cb819bfc4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 15:56:39 +0000
Subject: [PATCH 082/447] Update stage name and need to comment out build

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3c31ee09c..7709e4008 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
-            sh 'sorc/build_all.sh'
+            //sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
             script {
               case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
@@ -34,7 +34,7 @@ pipeline {
           }
         }
  
-        stage('Create Cases') {
+        stage('Create Experiments') {
         agent{ label 'orion-emc'}
             steps {
                 sh '''
-- 
GitLab


From 28bd6ec88d0cc1423fa379e5386a000624efb3d6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 16:07:54 +0000
Subject: [PATCH 083/447] removed echo of HOMEgfs in get_pr_case_list

---
 Jenkinsfile                  | 2 +-
 ci/scripts/utils/ci_utils.sh | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7709e4008..2e096946e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -28,7 +28,7 @@ pipeline {
             sh 'sorc/link_workflow.sh'
             script {
               case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              cases = case_list_output.tokenize('\n').remove(0)
+              cases = case_list_output.tokenize('\n')
               echo "cases: ${cases}"
             }
           }
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index cc9f8aaa2..bec983bc8 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -29,7 +29,6 @@ function get_pr_case_list () {
     # loop over every yaml file in the PR's ci/cases
     # and create an run directory for each one for this PR loop
     #############################################################
-    echo ${HOMEgfs}
     for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do
       case=$(basename "${yaml_config}" .yaml) || true
       echo "${case}"
-- 
GitLab


From a4206f42a0a14a87210b60ea992109ff1f0cabbd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:18:59 +0000
Subject: [PATCH 084/447] abstracted env setup of experment creatation

---
 Jenkinsfile                          | 24 ++++--------------------
 ci/scripts/utils/ci_utils_wrapper.sh |  1 +
 2 files changed, 5 insertions(+), 20 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2e096946e..7740601de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -29,7 +29,6 @@ pipeline {
             script {
               case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
               cases = case_list_output.tokenize('\n')
-              echo "cases: ${cases}"
             }
           }
         }
@@ -37,32 +36,17 @@ pipeline {
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             steps {
-                sh '''
-                   rm -rf ${RUNTESTS}
-                   mkdir -p ${RUNTESTS}
-                   '''
+                sh 'mkdir -p ${RUNTESTS}'
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
                     pullRequest.addLabel('CI-Orion-Running')
-                    echo "cases: ${cases}"
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
-                            script {
-                                env.case = case_name
-                            }
-                            sh '''
-                               source ci/platforms/config.orion
-                               pr_sha=$(git rev-parse --short HEAD)
-                               export pslot=${case}_${pr_sha}
-                               source workflow/gw_setup.sh
-                                HOMEgfs=${WORKSPACE}
-                                workflow/create_experiment.py --yaml ci/cases/pr/${case}.yaml
-                                '''
-                            script {
-                                  pullRequest.comment("SUCCESS creating ${case_name} on Orion")
-                            }
+                            script { env.case = case_name }
+                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
             }
         }
diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh
index f7cb0fb89..51c392fb9 100755
--- a/ci/scripts/utils/ci_utils_wrapper.sh
+++ b/ci/scripts/utils/ci_utils_wrapper.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+source "${HOMEgfs}/ush/detect_machine.sh"
 
 utitilty_function="${1}"
 
-- 
GitLab


From 009a0489df45371535ee7707b1c60abced5c2974 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:21:49 +0000
Subject: [PATCH 085/447] forgot to add updated ci_utils.sh

---
 ci/scripts/utils/ci_utils.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index bec983bc8..75ea04d08 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -33,4 +33,15 @@ function get_pr_case_list () {
       case=$(basename "${yaml_config}" .yaml) || true
       echo "${case}"
     done
+}
+
+function create_experiment () {
+  
+  yaml_config="${1}"
+  source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+  pr_sha=$(git rev-parse --short HEAD)
+  export pslot=${case}_${pr_sha}
+  source "${HOMEgfs}/workflow/gw_setup.sh"
+  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+
 }
\ No newline at end of file
-- 
GitLab


From ceb9db5c52f2f4c563fcb4a1f5416522e8e203e4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:31:04 +0000
Subject: [PATCH 086/447] triing to add delcrative steps in creating
 experiments again

---
 Jenkinsfile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7740601de..5c5f0bdd2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,4 @@
 
-def cases="cases"
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -26,10 +25,6 @@ pipeline {
           steps {
             //sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
-            script {
-              case_list_output = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-              cases = case_list_output.tokenize('\n')
-            }
           }
         }
  
@@ -40,10 +35,15 @@ pipeline {
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
                     pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
-                            script { env.case = case_name }
-                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                            agent{ label 'orion-emc'}
+                            steps {
+                              script { env.case = case_name }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                            }  
                         }
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-- 
GitLab


From 29492b32c8cea6fe44feefbdbc1a63853dd3c906 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:51:01 +0000
Subject: [PATCH 087/447] try stage stage

---
 Jenkinsfile | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5c5f0bdd2..79ec65d94 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -25,32 +25,29 @@ pipeline {
           steps {
             //sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
+            sh 'mkdir -p ${RUNTESTS}'
+            script {
+                pullRequest.removeLabel('CI-Orion-Building')
+                pullRequest.addLabel('CI-Orion-Running')
+                case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                cases = case_list.tokenize('\n')
+            }
           }
         }
  
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
-            steps {
-                sh 'mkdir -p ${RUNTESTS}'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                            steps {
-                              script { env.case = case_name }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                            }  
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+            cases.each { case_name ->
+                stage("Create ${case_name}") {
+                agent{ label 'orion-emc'}
+                    steps {
+                      script { env.case = case_name }
+                      sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                    }  
                 }
             }
+            script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
-    }    
 
     post {
         success {
-- 
GitLab


From 4f48ed4f671f1fcbb75decd2eed3c7d3d7cee826 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:53:47 +0000
Subject: [PATCH 088/447] add parallel

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 79ec65d94..0a5e2534a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,6 +38,7 @@ pipeline {
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             cases.each { case_name ->
+              parallel {
                 stage("Create ${case_name}") {
                 agent{ label 'orion-emc'}
                     steps {
@@ -45,6 +46,7 @@ pipeline {
                       sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                     }  
                 }
+              }
             }
             script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
-- 
GitLab


From 7c788526cd78b3984ec67c1da57c53c43566295e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 17:58:21 +0000
Subject: [PATCH 089/447] missed bracker over all staages

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0a5e2534a..3ebceffa4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,6 +50,7 @@ pipeline {
             }
             script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
+    }
 
     post {
         success {
-- 
GitLab


From ce4fa1ea2d1750ac3e378390af64d4657c4ce2f5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:19:56 +0000
Subject: [PATCH 090/447] added parllel over cases

---
 Jenkinsfile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3ebceffa4..0d81e34c8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,16 +38,16 @@ pipeline {
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             cases.each { case_name ->
-              parallel {
                 stage("Create ${case_name}") {
-                agent{ label 'orion-emc'}
-                    steps {
-                      script { env.case = case_name }
-                      sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                    }  
+                   cases[ "${case_name}"] = {
+                        node(label 'orion-emc') {
+                           script { env.case = case_name }
+                           sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                        }
+                    }
                 }
-              }
             }
+            parallel cases
             script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
     }
-- 
GitLab


From 994f0c642650827d2920acd9da376637d1592ddb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:21:56 +0000
Subject: [PATCH 091/447] replace node with agent

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0d81e34c8..8e17a8473 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
             cases.each { case_name ->
                 stage("Create ${case_name}") {
                    cases[ "${case_name}"] = {
-                        node(label 'orion-emc') {
+                        agent(label 'orion-emc') {
                            script { env.case = case_name }
                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
-- 
GitLab


From d1ae8d3d450825aefa7785ed49f7881324391505 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:24:17 +0000
Subject: [PATCH 092/447] removed agent block

---
 Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8e17a8473..d405fc2a9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -39,11 +39,10 @@ pipeline {
         agent{ label 'orion-emc'}
             cases.each { case_name ->
                 stage("Create ${case_name}") {
+                agent{ label 'orion-emc'}
                    cases[ "${case_name}"] = {
-                        agent(label 'orion-emc') {
-                           script { env.case = case_name }
-                           sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                        }
+                        script { env.case = case_name }
+                        sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                     }
                 }
             }
-- 
GitLab


From c1fb33626879da5658134754dd0f39e5800446ea Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:35:04 +0000
Subject: [PATCH 093/447] try global mapp

---
 Jenkinsfile | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d405fc2a9..5ebc68084 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,3 +1,4 @@
+define cases_map = [:]
 
 pipeline {
     agent{ label 'orion-emc'}
@@ -37,16 +38,18 @@ pipeline {
  
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
+            script {
             cases.each { case_name ->
                 stage("Create ${case_name}") {
                 agent{ label 'orion-emc'}
-                   cases[ "${case_name}"] = {
+                   cases_map[ "${case_name}"] = {
                         script { env.case = case_name }
                         sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                     }
                 }
             }
-            parallel cases
+            }
+            parallel cases_map
             script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
     }
-- 
GitLab


From e1902ca757723ab33b555d9cab9b1b7288b11560 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:37:46 +0000
Subject: [PATCH 094/447] added steps

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5ebc68084..426048b78 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,6 +38,7 @@ pipeline {
  
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
+            steps {
             script {
             cases.each { case_name ->
                 stage("Create ${case_name}") {
@@ -49,6 +50,7 @@ pipeline {
                 }
             }
             }
+            }
             parallel cases_map
             script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
-- 
GitLab


From b0f75b3403b278d3fd0e601c3d4e01b4453802bb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:44:30 +0000
Subject: [PATCH 095/447] revert to what had worked

---
 Jenkinsfile | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 426048b78..5c5f0bdd2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,4 +1,3 @@
-define cases_map = [:]
 
 pipeline {
     agent{ label 'orion-emc'}
@@ -26,35 +25,32 @@ pipeline {
           steps {
             //sh 'sorc/build_all.sh'
             sh 'sorc/link_workflow.sh'
-            sh 'mkdir -p ${RUNTESTS}'
-            script {
-                pullRequest.removeLabel('CI-Orion-Building')
-                pullRequest.addLabel('CI-Orion-Running')
-                case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                cases = case_list.tokenize('\n')
-            }
           }
         }
  
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             steps {
-            script {
-            cases.each { case_name ->
-                stage("Create ${case_name}") {
-                agent{ label 'orion-emc'}
-                   cases_map[ "${case_name}"] = {
-                        script { env.case = case_name }
-                        sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                sh 'mkdir -p ${RUNTESTS}'
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
+                    cases.each { case_name ->
+                        stage("Create ${case_name}") {
+                            agent{ label 'orion-emc'}
+                            steps {
+                              script { env.case = case_name }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                            }  
+                        }
                     }
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
             }
-            }
-            }
-            parallel cases_map
-            script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
         }
-    }
+    }    
 
     post {
         success {
-- 
GitLab


From 56ee98b84f0f15d9489c4ad4c7f84f249687698a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:47:54 +0000
Subject: [PATCH 096/447] remove steps

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5c5f0bdd2..ca9f83620 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,10 +40,8 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                             agent{ label 'orion-emc'}
-                            steps {
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                            }  
                         }
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-- 
GitLab


From 730272184f030894bc2b278801732a9af44b4bbb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 18:57:33 +0000
Subject: [PATCH 097/447] try parallel

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index ca9f83620..c53d6fda5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,11 +38,13 @@ pipeline {
                     case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
                     cases = case_list.tokenize('\n')
                     cases.each { case_name ->
+                        parallel {
                         stage("Create ${case_name}") {
                             agent{ label 'orion-emc'}
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
+                        }
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
-- 
GitLab


From e1abb2de8ba14a5b37642c762255643eaf49efb6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:04:57 +0000
Subject: [PATCH 098/447] try parallel run cases again

---
 Jenkinsfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c53d6fda5..00ce3e3ce 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,4 +1,6 @@
 
+def run_cases = [:]
+
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -38,7 +40,7 @@ pipeline {
                     case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
                     cases = case_list.tokenize('\n')
                     cases.each { case_name ->
-                        parallel {
+                        run_cases["${case_name}"] = {
                         stage("Create ${case_name}") {
                             agent{ label 'orion-emc'}
                               script { env.case = case_name }
@@ -48,6 +50,7 @@ pipeline {
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
+               parallel run_cases 
             }
         }
     }    
-- 
GitLab


From a793bb81b7126612128c35617fd3a9f5dd623aaf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:08:06 +0000
Subject: [PATCH 099/447] try parallel run cases again still

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 00ce3e3ce..680b18a2a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,8 +40,8 @@ pipeline {
                     case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
                     cases = case_list.tokenize('\n')
                     cases.each { case_name ->
-                        run_cases["${case_name}"] = {
                         stage("Create ${case_name}") {
+                        run_cases["${case_name}"] = {
                             agent{ label 'orion-emc'}
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-- 
GitLab


From ef8ee86e33367ca4fb5437dc01ee499de6d17cf0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:10:17 +0000
Subject: [PATCH 100/447] try parallel run cases again still 2

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 680b18a2a..9ec1ddc2f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,8 +49,8 @@ pipeline {
                         }
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+                    parallel run_cases 
                 }
-               parallel run_cases 
             }
         }
     }    
-- 
GitLab


From 0b7313cba905757ad52af88c3a61f7cbdf77ad8d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:18:18 +0000
Subject: [PATCH 101/447] move parallel

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9ec1ddc2f..8f0dd19e3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,15 +42,14 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                         run_cases["${case_name}"] = {
-                            agent{ label 'orion-emc'}
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                         }
                     }
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                    parallel run_cases 
                 }
+                parallel run_cases 
             }
         }
     }    
-- 
GitLab


From 27bf59190a2318033008d28723f87107c0b4ff0e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:21:15 +0000
Subject: [PATCH 102/447] move parallel yet again

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8f0dd19e3..ae87b3235 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,8 +42,10 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                         run_cases["${case_name}"] = {
+                              node('case-creator') {
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                              }
                         }
                         }
                     }
-- 
GitLab


From 41f00a9fd04d263bc0705b42ea9db989b151070b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:23:26 +0000
Subject: [PATCH 103/447] move parallel yet again

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ae87b3235..0e7692097 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,9 +49,9 @@ pipeline {
                         }
                         }
                     }
+                    parallel run_cases 
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
-                parallel run_cases 
             }
         }
     }    
-- 
GitLab


From 3a4c9bec6371b8c9ad8d8ea4695fd092aaa953e3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:34:41 +0000
Subject: [PATCH 104/447] agent node

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0e7692097..6d37ca7cc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,10 +42,10 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                         run_cases["${case_name}"] = {
-                              node('case-creator') {
+                              agent { node { label 'case-creator'} {
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                              }
+                              } 
                         }
                         }
                     }
-- 
GitLab


From 6b11fc5c3703c0ee4df90391032a9960a06a140a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:37:26 +0000
Subject: [PATCH 105/447] agent node

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6d37ca7cc..2e4443213 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,12 +42,11 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                         run_cases["${case_name}"] = {
-                              agent { node { label 'case-creator'} {
+                              agent { node { label 'case-creator'} }
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                               } 
                         }
-                        }
                     }
                     parallel run_cases 
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-- 
GitLab


From 1b18b814b7b2b1f013c555e834f12961b1d8af62 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 19:48:24 +0000
Subject: [PATCH 106/447] working version

---
 Jenkinsfile | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2e4443213..ca9f83620 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,4 @@
 
-def run_cases = [:]
-
 pipeline {
     agent{ label 'orion-emc'}
 
@@ -41,14 +39,11 @@ pipeline {
                     cases = case_list.tokenize('\n')
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
-                        run_cases["${case_name}"] = {
-                              agent { node { label 'case-creator'} }
+                            agent{ label 'orion-emc'}
                               script { env.case = case_name }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                              } 
                         }
                     }
-                    parallel run_cases 
                     script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
                 }
             }
-- 
GitLab


From 07ff62c4e0e6182137f9746e9c75b2ba1acb0e29 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 20:04:36 +0000
Subject: [PATCH 107/447] added get pslot list

---
 ci/scripts/utils/ci_utils.sh | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 75ea04d08..8f75b7ef6 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -35,6 +35,20 @@ function get_pr_case_list () {
     done
 }
 
+function get_pslot_list () {
+
+    local RUNTESTS="${1}"
+  
+    #############################################################
+    # loop over every yaml file in the PR's ci/cases
+    # and create an run directory for each one for this PR loop
+    #############################################################
+    for pslot in "${RUNTESTS}/EXPDIR/"*; do
+      echo "${pslot}"
+    done
+
+}
+
 function create_experiment () {
   
   yaml_config="${1}"
-- 
GitLab


From 8c1cd31da07a3b50e1063bf8ac6e9fd30b003a22 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 20:26:33 +0000
Subject: [PATCH 108/447] added (sequential) run experments

---
 Jenkinsfile                  | 18 ++++++++++++++++++
 ci/scripts/run-check_ci.sh   |  2 +-
 ci/scripts/utils/ci_utils.sh |  3 ++-
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ca9f83620..9ba74ca78 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,6 +48,24 @@ pipeline {
                 }
             }
         }
+
+        stage('Run Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                script {
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list", returnStdout: true ).trim()
+                    experiments = experiment_list.tokenize('\n')
+                    experiments.each { experiment_name ->
+                        stage("Run ${experiment_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.experiment = experiment_name }
+                              sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
+                }
+            }
+        }
     }    
 
     post {
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index a5c5369ef..939ac1b44 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -21,7 +21,7 @@ pslot=${2:-${pslot:-?}}        # Name of the experiment being tested by this scr
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
-HOMEgfs="${TEST_DIR}/HOMEgfs"
+HOMEgfs="${TEST_DIR}"
 RUNTESTS="${TEST_DIR}/RUNTESTS"
 
 # Source modules and setup logging
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 8f75b7ef6..120df5b5c 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -43,7 +43,8 @@ function get_pslot_list () {
     # loop over every yaml file in the PR's ci/cases
     # and create an run directory for each one for this PR loop
     #############################################################
-    for pslot in "${RUNTESTS}/EXPDIR/"*; do
+    for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do
+      pslot=$(basename "${plsot_dir}") || true
       echo "${pslot}"
     done
 
-- 
GitLab


From deb4372567bc475f55dfb209170f21d5247904f9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 20:28:39 +0000
Subject: [PATCH 109/447] added build all script in build all for real

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9ba74ca78..3742d19da 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -23,7 +23,7 @@ pipeline {
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
-            //sh 'sorc/build_all.sh'
+            sh 'sorc/build_all.sh -gu'
             sh 'sorc/link_workflow.sh'
           }
         }
-- 
GitLab


From 6409a337719ffa8fe8d0011fd4c32f09273793a2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 28 Dec 2023 22:06:57 +0000
Subject: [PATCH 110/447] forgot path for pslot list

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3742d19da..b51033d22 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -23,7 +23,7 @@ pipeline {
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
-            sh 'sorc/build_all.sh -gu'
+            //sh 'sorc/build_all.sh -gu'
             sh 'sorc/link_workflow.sh'
           }
         }
@@ -53,7 +53,7 @@ pipeline {
         agent{ label 'orion-emc'}
             steps {
                 script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list", returnStdout: true ).trim()
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim()
                     experiments = experiment_list.tokenize('\n')
                     experiments.each { experiment_name ->
                         stage("Run ${experiment_name}") {
-- 
GitLab


From 5c2d107290f24e7f7c052f5b83449870f4fe0925 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 18:17:21 +0000
Subject: [PATCH 111/447] clear logs on controler by renaming Jenkinsfile

---
 Jenkinsfile => Jenkinsfile_org | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile => Jenkinsfile_org (100%)

diff --git a/Jenkinsfile b/Jenkinsfile_org
similarity index 100%
rename from Jenkinsfile
rename to Jenkinsfile_org
-- 
GitLab


From 04913f551861eba109b86e72de974b1b2e9d8f48 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 18:35:25 +0000
Subject: [PATCH 112/447] added case in create experiment

---
 ci/scripts/utils/ci_utils.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 120df5b5c..ab410347a 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -51,12 +51,16 @@ function get_pslot_list () {
 }
 
 function create_experiment () {
+
+  local yaml_config="${1}"
+  cd "${HOMEgfs}" || exit 1
+  pr_sha=$(git rev-parse --short HEAD)
+  case=$(basename "${yaml_config}" .yaml) || true
   
-  yaml_config="${1}"
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
-  pr_sha=$(git rev-parse --short HEAD)
-  export pslot=${case}_${pr_sha}
   source "${HOMEgfs}/workflow/gw_setup.sh"
+
+  export pslot=${case}_${pr_sha}
   "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
\ No newline at end of file
-- 
GitLab


From 184045b963be70532c32677f0a06dc8644263ea7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 19:40:26 +0000
Subject: [PATCH 113/447] add Jenkisfile back for new log

---
 Jenkinsfile_org => Jenkinsfile | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Jenkinsfile_org => Jenkinsfile (100%)

diff --git a/Jenkinsfile_org b/Jenkinsfile
similarity index 100%
rename from Jenkinsfile_org
rename to Jenkinsfile
-- 
GitLab


From fd1c7475b1115bc9ebc3c91fb4652d0fa37bc1b7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 19:52:56 +0000
Subject: [PATCH 114/447] typo bug with pslot in get_pslot_list and uncomment
 build

---
 Jenkinsfile_org              | 85 ++++++++++++++++++++++++++++++++++++
 ci/scripts/utils/ci_utils.sh |  2 +-
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 Jenkinsfile_org

diff --git a/Jenkinsfile_org b/Jenkinsfile_org
new file mode 100644
index 000000000..9c00a83f5
--- /dev/null
+++ b/Jenkinsfile_org
@@ -0,0 +1,85 @@
+
+pipeline {
+    agent{ label 'orion-emc'}
+
+    environment {
+        RUNTESTS = "${WORKSPACE}/RUNTESTS"
+    }
+
+    stages {
+
+        stage('Checkout') {
+        agent{ label 'orion-emc'}
+            steps {
+                checkout scm
+                script {
+                   pullRequest.removeLabel('CI-Orion-Ready')
+                   pullRequest.addLabel('CI-Orion-Building')
+                }
+                sh 'git submodule update --init --recursive'
+            }
+        }
+
+        stage('Build') {
+        agent{ label 'orion-emc'}
+          steps {
+            sh 'sorc/build_all.sh -gu'
+            sh 'sorc/link_workflow.sh'
+          }
+        }
+ 
+        stage('Create Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                sh 'mkdir -p ${RUNTESTS}'
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
+                    cases.each { case_name ->
+                        stage("Create ${case_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.case = case_name }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+                }
+            }
+        }
+
+        stage('Run Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                script {
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim()
+                    experiments = experiment_list.tokenize('\n')
+                    experiments.each { experiment_name ->
+                        stage("Run ${experiment_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.experiment = experiment_name }
+                              sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
+                }
+            }
+        }
+    }    
+
+    post {
+        success {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Passed')  
+            }
+        }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Failed')  
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index ab410347a..d46688bd6 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -44,7 +44,7 @@ function get_pslot_list () {
     # and create an run directory for each one for this PR loop
     #############################################################
     for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do
-      pslot=$(basename "${plsot_dir}") || true
+      pslot=$(basename "${pslot_dir}") || true
       echo "${pslot}"
     done
 
-- 
GitLab


From 208117d82da6fdfb81450aa19ea1e8108deb4eff Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 20:02:21 +0000
Subject: [PATCH 115/447] bug misspelled ci run script

---
 Jenkinsfile     |  2 +-
 Jenkinsfile_org | 85 -------------------------------------------------
 2 files changed, 1 insertion(+), 86 deletions(-)
 delete mode 100644 Jenkinsfile_org

diff --git a/Jenkinsfile b/Jenkinsfile
index b51033d22..d16695d55 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -59,7 +59,7 @@ pipeline {
                         stage("Run ${experiment_name}") {
                             agent{ label 'orion-emc'}
                               script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}'
+                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
                         }
                     }
                     script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
diff --git a/Jenkinsfile_org b/Jenkinsfile_org
deleted file mode 100644
index 9c00a83f5..000000000
--- a/Jenkinsfile_org
+++ /dev/null
@@ -1,85 +0,0 @@
-
-pipeline {
-    agent{ label 'orion-emc'}
-
-    environment {
-        RUNTESTS = "${WORKSPACE}/RUNTESTS"
-    }
-
-    stages {
-
-        stage('Checkout') {
-        agent{ label 'orion-emc'}
-            steps {
-                checkout scm
-                script {
-                   pullRequest.removeLabel('CI-Orion-Ready')
-                   pullRequest.addLabel('CI-Orion-Building')
-                }
-                sh 'git submodule update --init --recursive'
-            }
-        }
-
-        stage('Build') {
-        agent{ label 'orion-emc'}
-          steps {
-            sh 'sorc/build_all.sh -gu'
-            sh 'sorc/link_workflow.sh'
-          }
-        }
- 
-        stage('Create Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                sh 'mkdir -p ${RUNTESTS}'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.case = case_name }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                }
-            }
-        }
-
-        stage('Run Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim()
-                    experiments = experiment_list.tokenize('\n')
-                    experiments.each { experiment_name ->
-                        stage("Run ${experiment_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check.sh ${WORKSPACE} ${experiment}'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
-                }
-            }
-        }
-    }    
-
-    post {
-        success {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Passed')  
-            }
-        }
-        failure {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Failed')  
-            }
-        }
-    }
-}
\ No newline at end of file
-- 
GitLab


From ef880c844e1112e04daefda927e661cabb09c8d2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 20:07:40 +0000
Subject: [PATCH 116/447] added rm RUNTESTS dir before createing experminets at
 the pipeline level

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index d16695d55..db0609a40 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -31,6 +31,7 @@ pipeline {
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             steps {
+                sh 'rm -Rf ${RUNTESTS}'
                 sh 'mkdir -p ${RUNTESTS}'
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
-- 
GitLab


From dcc97f2ae099a779403fdfb218852a32f1debbee Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 21:07:14 +0000
Subject: [PATCH 117/447] removed global RUNTESTS, was getting wrong WORKSPACE
 dir

---
 Jenkinsfile | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index db0609a40..05064b6e6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,10 +2,6 @@
 pipeline {
     agent{ label 'orion-emc'}
 
-    environment {
-        RUNTESTS = "${WORKSPACE}/RUNTESTS"
-    }
-
     stages {
 
         stage('Checkout') {
@@ -23,7 +19,7 @@ pipeline {
         stage('Build') {
         agent{ label 'orion-emc'}
           steps {
-            //sh 'sorc/build_all.sh -gu'
+            sh 'sorc/build_all.sh -gu'
             sh 'sorc/link_workflow.sh'
           }
         }
@@ -41,7 +37,10 @@ pipeline {
                     cases.each { case_name ->
                         stage("Create ${case_name}") {
                             agent{ label 'orion-emc'}
-                              script { env.case = case_name }
+                              script { 
+                                env.case = case_name
+                                env.RUNTESTS = "${WORKSPCE}/RUNTESTS"
+                              }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
@@ -54,7 +53,7 @@ pipeline {
         agent{ label 'orion-emc'}
             steps {
                 script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${RUNTESTS}", returnStdout: true ).trim()
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
                     experiments = experiment_list.tokenize('\n')
                     experiments.each { experiment_name ->
                         stage("Run ${experiment_name}") {
-- 
GitLab


From 3a63a905d2b177d16b427bfc96b9b54937a44163 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 29 Dec 2023 22:18:16 +0000
Subject: [PATCH 118/447] removed RUNTEST var in one more place

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 05064b6e6..a742f3609 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -27,8 +27,8 @@ pipeline {
         stage('Create Experiments') {
         agent{ label 'orion-emc'}
             steps {
-                sh 'rm -Rf ${RUNTESTS}'
-                sh 'mkdir -p ${RUNTESTS}'
+                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
+                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
                 script {
                     pullRequest.removeLabel('CI-Orion-Building')
                     pullRequest.addLabel('CI-Orion-Running')
-- 
GitLab


From b367a1c22073deb91b60c82c4a5f1a040f792dbf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 30 Dec 2023 00:07:53 +0000
Subject: [PATCH 119/447] SPELLED WORKSPACE WRONG

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a742f3609..142c5bb0a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -39,7 +39,7 @@ pipeline {
                             agent{ label 'orion-emc'}
                               script { 
                                 env.case = case_name
-                                env.RUNTESTS = "${WORKSPCE}/RUNTESTS"
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
                               }
                               sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
-- 
GitLab


From b9c50edaba1f058c4bd148c5078222d65d39fd18 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 16:57:49 +0000
Subject: [PATCH 120/447] working with matrix construct test case

---
 Jenkinsfile      | 93 +++++++++++-------------------------------------
 Jenkinsfile_main | 85 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+), 73 deletions(-)
 create mode 100644 Jenkinsfile_main

diff --git a/Jenkinsfile b/Jenkinsfile
index 142c5bb0a..88dba0229 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,85 +1,32 @@
-
 pipeline {
-    agent{ label 'orion-emc'}
-
+    agent none
     stages {
-
-        stage('Checkout') {
-        agent{ label 'orion-emc'}
-            steps {
-                checkout scm
-                script {
-                   pullRequest.removeLabel('CI-Orion-Ready')
-                   pullRequest.addLabel('CI-Orion-Building')
+        stage('BuildAndTest') {
+            matrix {
+                agent any
+                axes {
+                    axis {
+                        name 'PLATFORM'
+                        values 'Orion', 'Hera', 'Hercules'
+                    }
+                    axis {
+                        name 'Cases'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
+                    }
                 }
-                sh 'git submodule update --init --recursive'
-            }
-        }
-
-        stage('Build') {
-        agent{ label 'orion-emc'}
-          steps {
-            sh 'sorc/build_all.sh -gu'
-            sh 'sorc/link_workflow.sh'
-          }
-        }
- 
-        stage('Create Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
-                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                              script { 
-                                env.case = case_name
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
-                              }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                stages {
+                    stage('Build') {
+                        steps {
+                            echo "Do Build for ${PLATFORM}"
                         }
                     }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                }
-            }
-        }
-
-        stage('Run Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
-                    experiments = experiment_list.tokenize('\n')
-                    experiments.each { experiment_name ->
-                        stage("Run ${experiment_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
+                    stage('Run Tests') {
+                        steps {
+                            echo "Do Test for ${PLATFORM} - ${Cases}"
                         }
                     }
-                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
                 }
             }
         }
-    }    
-
-    post {
-        success {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Passed')  
-            }
-        }
-        failure {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Failed')  
-            }
-        }
     }
 }
\ No newline at end of file
diff --git a/Jenkinsfile_main b/Jenkinsfile_main
new file mode 100644
index 000000000..142c5bb0a
--- /dev/null
+++ b/Jenkinsfile_main
@@ -0,0 +1,85 @@
+
+pipeline {
+    agent{ label 'orion-emc'}
+
+    stages {
+
+        stage('Checkout') {
+        agent{ label 'orion-emc'}
+            steps {
+                checkout scm
+                script {
+                   pullRequest.removeLabel('CI-Orion-Ready')
+                   pullRequest.addLabel('CI-Orion-Building')
+                }
+                sh 'git submodule update --init --recursive'
+            }
+        }
+
+        stage('Build') {
+        agent{ label 'orion-emc'}
+          steps {
+            sh 'sorc/build_all.sh -gu'
+            sh 'sorc/link_workflow.sh'
+          }
+        }
+ 
+        stage('Create Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
+                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
+                    cases.each { case_name ->
+                        stage("Create ${case_name}") {
+                            agent{ label 'orion-emc'}
+                              script { 
+                                env.case = case_name
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                              }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+                }
+            }
+        }
+
+        stage('Run Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                script {
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
+                    experiments = experiment_list.tokenize('\n')
+                    experiments.each { experiment_name ->
+                        stage("Run ${experiment_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.experiment = experiment_name }
+                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
+                }
+            }
+        }
+    }    
+
+    post {
+        success {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Passed')  
+            }
+        }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Failed')  
+            }
+        }
+    }
+}
\ No newline at end of file
-- 
GitLab


From d4219d9fd22a3b7528ac60c1180530a70f761d3a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 17:30:27 +0000
Subject: [PATCH 121/447] added correct node name for agent

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 88dba0229..dbdcb3918 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,7 +3,7 @@ pipeline {
     stages {
         stage('BuildAndTest') {
             matrix {
-                agent any
+                agent{ label 'orion-emc'}
                 axes {
                     axis {
                         name 'PLATFORM'
-- 
GitLab


From 56bd357ea8a55a1274773ad37a532711fe47c676 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 17:33:27 +0000
Subject: [PATCH 122/447] added correct node name for agent in outer loop too

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dbdcb3918..bedaf953f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,5 @@
 pipeline {
-    agent none
+    agent{ label 'orion-emc'}
     stages {
         stage('BuildAndTest') {
             matrix {
-- 
GitLab


From 7e8d366683329e973f4ee1cbc51af2394f306cfe Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 17:36:12 +0000
Subject: [PATCH 123/447] move to working version

---
 Jenkinsfile      | 91 ++++++++++++++++++++++++++++++++++++++----------
 Jenkinsfile_main | 85 --------------------------------------------
 Jenkinsfile_temp | 32 +++++++++++++++++
 3 files changed, 104 insertions(+), 104 deletions(-)
 delete mode 100644 Jenkinsfile_main
 create mode 100644 Jenkinsfile_temp

diff --git a/Jenkinsfile b/Jenkinsfile
index bedaf953f..142c5bb0a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,32 +1,85 @@
+
 pipeline {
     agent{ label 'orion-emc'}
+
     stages {
-        stage('BuildAndTest') {
-            matrix {
-                agent{ label 'orion-emc'}
-                axes {
-                    axis {
-                        name 'PLATFORM'
-                        values 'Orion', 'Hera', 'Hercules'
-                    }
-                    axis {
-                        name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
-                    }
+
+        stage('Checkout') {
+        agent{ label 'orion-emc'}
+            steps {
+                checkout scm
+                script {
+                   pullRequest.removeLabel('CI-Orion-Ready')
+                   pullRequest.addLabel('CI-Orion-Building')
                 }
-                stages {
-                    stage('Build') {
-                        steps {
-                            echo "Do Build for ${PLATFORM}"
+                sh 'git submodule update --init --recursive'
+            }
+        }
+
+        stage('Build') {
+        agent{ label 'orion-emc'}
+          steps {
+            sh 'sorc/build_all.sh -gu'
+            sh 'sorc/link_workflow.sh'
+          }
+        }
+ 
+        stage('Create Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
+                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
+                    cases.each { case_name ->
+                        stage("Create ${case_name}") {
+                            agent{ label 'orion-emc'}
+                              script { 
+                                env.case = case_name
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                              }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
-                    stage('Run Tests') {
-                        steps {
-                            echo "Do Test for ${PLATFORM} - ${Cases}"
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+                }
+            }
+        }
+
+        stage('Run Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                script {
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
+                    experiments = experiment_list.tokenize('\n')
+                    experiments.each { experiment_name ->
+                        stage("Run ${experiment_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.experiment = experiment_name }
+                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
                         }
                     }
+                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
                 }
             }
         }
+    }    
+
+    post {
+        success {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Passed')  
+            }
+        }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Failed')  
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/Jenkinsfile_main b/Jenkinsfile_main
deleted file mode 100644
index 142c5bb0a..000000000
--- a/Jenkinsfile_main
+++ /dev/null
@@ -1,85 +0,0 @@
-
-pipeline {
-    agent{ label 'orion-emc'}
-
-    stages {
-
-        stage('Checkout') {
-        agent{ label 'orion-emc'}
-            steps {
-                checkout scm
-                script {
-                   pullRequest.removeLabel('CI-Orion-Ready')
-                   pullRequest.addLabel('CI-Orion-Building')
-                }
-                sh 'git submodule update --init --recursive'
-            }
-        }
-
-        stage('Build') {
-        agent{ label 'orion-emc'}
-          steps {
-            sh 'sorc/build_all.sh -gu'
-            sh 'sorc/link_workflow.sh'
-          }
-        }
- 
-        stage('Create Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
-                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                              script { 
-                                env.case = case_name
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
-                              }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                }
-            }
-        }
-
-        stage('Run Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
-                    experiments = experiment_list.tokenize('\n')
-                    experiments.each { experiment_name ->
-                        stage("Run ${experiment_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
-                }
-            }
-        }
-    }    
-
-    post {
-        success {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Passed')  
-            }
-        }
-        failure {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Failed')  
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/Jenkinsfile_temp b/Jenkinsfile_temp
new file mode 100644
index 000000000..bedaf953f
--- /dev/null
+++ b/Jenkinsfile_temp
@@ -0,0 +1,32 @@
+pipeline {
+    agent{ label 'orion-emc'}
+    stages {
+        stage('BuildAndTest') {
+            matrix {
+                agent{ label 'orion-emc'}
+                axes {
+                    axis {
+                        name 'PLATFORM'
+                        values 'Orion', 'Hera', 'Hercules'
+                    }
+                    axis {
+                        name 'Cases'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
+                    }
+                }
+                stages {
+                    stage('Build') {
+                        steps {
+                            echo "Do Build for ${PLATFORM}"
+                        }
+                    }
+                    stage('Run Tests') {
+                        steps {
+                            echo "Do Test for ${PLATFORM} - ${Cases}"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
\ No newline at end of file
-- 
GitLab


From 1d81eba34679906efb81bee6eba2504067396760 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 17:47:39 +0000
Subject: [PATCH 124/447] back to testing matrix locally

---
 Jenkinsfile      | 91 ++++++++++--------------------------------------
 Jenkinsfile_main | 85 ++++++++++++++++++++++++++++++++++++++++++++
 Jenkinsfile_temp | 32 -----------------
 3 files changed, 104 insertions(+), 104 deletions(-)
 create mode 100644 Jenkinsfile_main
 delete mode 100644 Jenkinsfile_temp

diff --git a/Jenkinsfile b/Jenkinsfile
index 142c5bb0a..bedaf953f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,85 +1,32 @@
-
 pipeline {
     agent{ label 'orion-emc'}
-
     stages {
-
-        stage('Checkout') {
-        agent{ label 'orion-emc'}
-            steps {
-                checkout scm
-                script {
-                   pullRequest.removeLabel('CI-Orion-Ready')
-                   pullRequest.addLabel('CI-Orion-Building')
+        stage('BuildAndTest') {
+            matrix {
+                agent{ label 'orion-emc'}
+                axes {
+                    axis {
+                        name 'PLATFORM'
+                        values 'Orion', 'Hera', 'Hercules'
+                    }
+                    axis {
+                        name 'Cases'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
+                    }
                 }
-                sh 'git submodule update --init --recursive'
-            }
-        }
-
-        stage('Build') {
-        agent{ label 'orion-emc'}
-          steps {
-            sh 'sorc/build_all.sh -gu'
-            sh 'sorc/link_workflow.sh'
-          }
-        }
- 
-        stage('Create Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
-                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                              script { 
-                                env.case = case_name
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
-                              }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                stages {
+                    stage('Build') {
+                        steps {
+                            echo "Do Build for ${PLATFORM}"
                         }
                     }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                }
-            }
-        }
-
-        stage('Run Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
-                    experiments = experiment_list.tokenize('\n')
-                    experiments.each { experiment_name ->
-                        stage("Run ${experiment_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
+                    stage('Run Tests') {
+                        steps {
+                            echo "Do Test for ${PLATFORM} - ${Cases}"
                         }
                     }
-                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
                 }
             }
         }
-    }    
-
-    post {
-        success {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Passed')  
-            }
-        }
-        failure {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Failed')  
-            }
-        }
     }
 }
\ No newline at end of file
diff --git a/Jenkinsfile_main b/Jenkinsfile_main
new file mode 100644
index 000000000..142c5bb0a
--- /dev/null
+++ b/Jenkinsfile_main
@@ -0,0 +1,85 @@
+
+pipeline {
+    agent{ label 'orion-emc'}
+
+    stages {
+
+        stage('Checkout') {
+        agent{ label 'orion-emc'}
+            steps {
+                checkout scm
+                script {
+                   pullRequest.removeLabel('CI-Orion-Ready')
+                   pullRequest.addLabel('CI-Orion-Building')
+                }
+                sh 'git submodule update --init --recursive'
+            }
+        }
+
+        stage('Build') {
+        agent{ label 'orion-emc'}
+          steps {
+            sh 'sorc/build_all.sh -gu'
+            sh 'sorc/link_workflow.sh'
+          }
+        }
+ 
+        stage('Create Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
+                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
+                script {
+                    pullRequest.removeLabel('CI-Orion-Building')
+                    pullRequest.addLabel('CI-Orion-Running')
+                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
+                    cases = case_list.tokenize('\n')
+                    cases.each { case_name ->
+                        stage("Create ${case_name}") {
+                            agent{ label 'orion-emc'}
+                              script { 
+                                env.case = case_name
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                              }
+                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
+                }
+            }
+        }
+
+        stage('Run Experiments') {
+        agent{ label 'orion-emc'}
+            steps {
+                script {
+                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
+                    experiments = experiment_list.tokenize('\n')
+                    experiments.each { experiment_name ->
+                        stage("Run ${experiment_name}") {
+                            agent{ label 'orion-emc'}
+                              script { env.experiment = experiment_name }
+                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
+                        }
+                    }
+                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
+                }
+            }
+        }
+    }    
+
+    post {
+        success {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Passed')  
+            }
+        }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-Orion-Running')
+                pullRequest.addLabel('CI-Orion-Failed')  
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/Jenkinsfile_temp b/Jenkinsfile_temp
deleted file mode 100644
index bedaf953f..000000000
--- a/Jenkinsfile_temp
+++ /dev/null
@@ -1,32 +0,0 @@
-pipeline {
-    agent{ label 'orion-emc'}
-    stages {
-        stage('BuildAndTest') {
-            matrix {
-                agent{ label 'orion-emc'}
-                axes {
-                    axis {
-                        name 'PLATFORM'
-                        values 'Orion', 'Hera', 'Hercules'
-                    }
-                    axis {
-                        name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
-                    }
-                }
-                stages {
-                    stage('Build') {
-                        steps {
-                            echo "Do Build for ${PLATFORM}"
-                        }
-                    }
-                    stage('Run Tests') {
-                        steps {
-                            echo "Do Test for ${PLATFORM} - ${Cases}"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
-- 
GitLab


From 56e50bd06d6738379e708b848b23108d94c57d71 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 18:07:22 +0000
Subject: [PATCH 125/447] removed inner agent and two platforms for testing

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bedaf953f..ebdd8bc74 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,11 +3,10 @@ pipeline {
     stages {
         stage('BuildAndTest') {
             matrix {
-                agent{ label 'orion-emc'}
                 axes {
                     axis {
                         name 'PLATFORM'
-                        values 'Orion', 'Hera', 'Hercules'
+                        values 'Orion', 'Hera'
                     }
                     axis {
                         name 'Cases'
-- 
GitLab


From 707336e78dce6ca3ea5e4f30bfaa617587e16d6c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 18:10:29 +0000
Subject: [PATCH 126/447] removed inner agent and two platforms for testing
 still

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ebdd8bc74..c804e2b7b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -14,12 +14,12 @@ pipeline {
                     }
                 }
                 stages {
-                    stage('Build') {
+                    stage('Build ${PLATFORM}') {
                         steps {
                             echo "Do Build for ${PLATFORM}"
                         }
                     }
-                    stage('Run Tests') {
+                    stage('Run Tests ${PLATFORM} - ${Cases}') {
                         steps {
                             echo "Do Test for ${PLATFORM} - ${Cases}"
                         }
-- 
GitLab


From b39af6152a1332def29c2e75ac2c8cd678453261 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 20:17:09 +0000
Subject: [PATCH 127/447] added sample pipeline for arch opt

---
 Jenkins_arch_opt | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 Jenkins_arch_opt

diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt
new file mode 100644
index 000000000..3aa31fa53
--- /dev/null
+++ b/Jenkins_arch_opt
@@ -0,0 +1,38 @@
+pipeline {
+    agent none
+
+    stages {
+        parallel {
+            // The matrix directive would only be available
+            // in parallel blocks and is used to generate stages in
+            // a declarative fashion
+            matrix {
+                // Options define a map of options we could potentially have,
+                // each map defines a matrix for each individual stage
+                opts: [
+                    [machine: 'emc-orion'],
+                    [machine: 'emc-hera'],
+                ]
+            }
+
+            // To access variables defined in our opts we can use the groovy
+            // standard `it`
+            stage("${it.machine} Building & Testing") {
+                agent { label "${it.machine}" }
+                stage("Build") {
+                    steps {
+                        sh("echo Build on ${it.machine}")
+                    }
+                }
+                stage("Run tests") {
+                    steps {
+                        sh("Running Test One  on ${it.machine}")
+                    }
+                    steps {
+                        sh("Running Test Two on ${it.machine}")
+                    }
+                }
+            }    
+        }
+    }
+}
\ No newline at end of file
-- 
GitLab


From f74ee472063f697a5d23ea10a00d2aa54d65c2ef Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 20:21:06 +0000
Subject: [PATCH 128/447] missed bracket

---
 Jenkins_arch_opt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt
index 3aa31fa53..f10858de7 100644
--- a/Jenkins_arch_opt
+++ b/Jenkins_arch_opt
@@ -28,9 +28,6 @@ pipeline {
                     steps {
                         sh("Running Test One  on ${it.machine}")
                     }
-                    steps {
-                        sh("Running Test Two on ${it.machine}")
-                    }
                 }
             }    
         }
-- 
GitLab


From eeb94e723bfed10265fc398259e27e72dba88fcf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 21:08:48 +0000
Subject: [PATCH 129/447] basic parallel matrix

---
 Jenkinsfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c804e2b7b..408d45e01 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,12 +1,13 @@
 pipeline {
-    agent{ label 'orion-emc'}
+    agent none
     stages {
         stage('BuildAndTest') {
             matrix {
+                agent { label '${PLATFORM}-emc' }
                 axes {
                     axis {
                         name 'PLATFORM'
-                        values 'Orion', 'Hera'
+                        values 'orion', 'hera'
                     }
                     axis {
                         name 'Cases'
@@ -14,12 +15,12 @@ pipeline {
                     }
                 }
                 stages {
-                    stage('Build ${PLATFORM}') {
+                    stage('Build') {
                         steps {
-                            echo "Do Build for ${PLATFORM}"
+                            echo "Do Build for ${env.PLATFORM}"
                         }
                     }
-                    stage('Run Tests ${PLATFORM} - ${Cases}') {
+                    stage('Run Cases') {
                         steps {
                             echo "Do Test for ${PLATFORM} - ${Cases}"
                         }
-- 
GitLab


From 95d100e9541a919bbc226dde41e4ab6acc81a5c5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 21:13:26 +0000
Subject: [PATCH 130/447] double quotes in label for platform

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 408d45e01..b7e7e153b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,7 +3,7 @@ pipeline {
     stages {
         stage('BuildAndTest') {
             matrix {
-                agent { label '${PLATFORM}-emc' }
+                agent { label "${PLATFORM}-emc" }
                 axes {
                     axis {
                         name 'PLATFORM'
@@ -17,7 +17,7 @@ pipeline {
                 stages {
                     stage('Build') {
                         steps {
-                            echo "Do Build for ${env.PLATFORM}"
+                            echo "Do Build for ${PLATFORM}"
                         }
                     }
                     stage('Run Cases') {
-- 
GitLab


From b78045cc3c999a92e1489964c2d1a18a3f044120 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 23:42:34 +0000
Subject: [PATCH 131/447] added label based agents in matrix

---
 Jenkinsfile | 36 ++++++++++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b7e7e153b..c67ee68fb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,14 +1,38 @@
 pipeline {
     agent none
     stages {
-        stage('BuildAndTest') {
+
+        stage( 'Get Machine' ) {
+
+            steps {
+                script {
+                    for (label in pullRequest.labels) {
+                        if ((label.matches("CI-Hera-Ready"))) {
+                             env.CHOICE_NODE='hera-emc'
+                        }  
+                        else if ((label.matches("CI-Orion-Ready"))) {
+                            env.CHOICE_NODE='orion-emc'
+                        }  
+                        else if ((label.matches("CI-Hercules-Ready"))) {
+                            env.CHOICE_NODE='hercules-emc'
+                        }  
+                        else { 
+                            env.CHOICE_NODE='none'
+                        }
+                     }
+                }
+            }
+        }
+
+        stage( 'Build and Test' ) {
+
+            when {
+                expression { env.CHOICE_NODE != 'none' }
+            }
+
             matrix {
-                agent { label "${PLATFORM}-emc" }
+                agent { label "${CHOICE_NODE}" }
                 axes {
-                    axis {
-                        name 'PLATFORM'
-                        values 'orion', 'hera'
-                    }
                     axis {
                         name 'Cases'
                         values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
-- 
GitLab


From 7946fad108bfe329a7ce0ae34b7c65b4ae937e4f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Jan 2024 23:47:16 +0000
Subject: [PATCH 132/447] removed PLATFORM replaced by CHOICE_NODE

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c67ee68fb..a70096bb7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,12 +41,12 @@ pipeline {
                 stages {
                     stage('Build') {
                         steps {
-                            echo "Do Build for ${PLATFORM}"
+                            echo "Do Build for ${CHOICE_NODE} - ${Cases}"
                         }
                     }
                     stage('Run Cases') {
                         steps {
-                            echo "Do Test for ${PLATFORM} - ${Cases}"
+                            echo "Do Test for ${CHOICE_NODE} - ${Cases}"
                         }
                     }
                 }
-- 
GitLab


From 639b4f68d69a1b5fe84d5e1741b464276a00c6a9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:02:31 +0000
Subject: [PATCH 133/447] added build and create experments steps

---
 Jenkinsfile | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index a70096bb7..5b52e7d98 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,12 @@
 pipeline {
     agent none
+
+    options {
+        disableConcurrentBuilds()
+        overrideIndexTriggers(false)
+        skipDefaultCheckout(true)
+    }
+
     stages {
 
         stage( 'Get Machine' ) {
@@ -42,6 +49,23 @@ pipeline {
                     stage('Build') {
                         steps {
                             echo "Do Build for ${CHOICE_NODE} - ${Cases}"
+                            cleaanWs()
+                            checkout scm
+                            sh 'sorc/build_all.sh -gu'
+                            sh 'sorc/link_workflow.sh'
+                        }
+                    }
+                    stage('Create Experiment') {
+                        steps {
+                            script { 
+                                env.case = ${Cases}
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                            }
+                            sh '''
+                                rm -rf ${RUNTESTS}
+                                mkdir -p ${RUNTESTS}
+                                '''
+                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
                     stage('Run Cases') {
-- 
GitLab


From 4df67d89802814b84eed06e4d06935cc59e822e5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:04:18 +0000
Subject: [PATCH 134/447] spelled clean wrong

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5b52e7d98..e3377c0be 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,7 +49,7 @@ pipeline {
                     stage('Build') {
                         steps {
                             echo "Do Build for ${CHOICE_NODE} - ${Cases}"
-                            cleaanWs()
+                            cleanWs()
                             checkout scm
                             sh 'sorc/build_all.sh -gu'
                             sh 'sorc/link_workflow.sh'
-- 
GitLab


From cdf2cb129f093e1879d6069241da68459ee31901 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:24:43 +0000
Subject: [PATCH 135/447] added MACHINE and updated PR Labels so next platform
 can run

---
 Jenkinsfile | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e3377c0be..b5990db10 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -15,16 +15,16 @@ pipeline {
                 script {
                     for (label in pullRequest.labels) {
                         if ((label.matches("CI-Hera-Ready"))) {
-                             env.CHOICE_NODE='hera-emc'
+                             env.MACHINE='hera'
                         }  
                         else if ((label.matches("CI-Orion-Ready"))) {
-                            env.CHOICE_NODE='orion-emc'
+                            env.MACHINE='orion'
                         }  
                         else if ((label.matches("CI-Hercules-Ready"))) {
-                            env.CHOICE_NODE='hercules-emc'
+                            env.MACHINE='hercules'
                         }  
                         else { 
-                            env.CHOICE_NODE='none'
+                            env.MACHINE='none'
                         }
                      }
                 }
@@ -34,11 +34,11 @@ pipeline {
         stage( 'Build and Test' ) {
 
             when {
-                expression { env.CHOICE_NODE != 'none' }
+                expression { env.MACHINE != 'none' }
             }
 
             matrix {
-                agent { label "${CHOICE_NODE}" }
+                agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name 'Cases'
@@ -48,10 +48,14 @@ pipeline {
                 stages {
                     stage('Build') {
                         steps {
-                            echo "Do Build for ${CHOICE_NODE} - ${Cases}"
+                           echo "Do Build for ${MACHINE^} - ${Cases}"
+                           script {
+                            pullRequest.removeLabel('CI-${MACHINE^}-Ready')
+                            pullRequest.addLabel('CI-${MACHINE^}-Building')
+                           }
                             cleanWs()
                             checkout scm
-                            sh 'sorc/build_all.sh -gu'
+                            //sh 'sorc/build_all.sh -gu'
                             sh 'sorc/link_workflow.sh'
                         }
                     }
@@ -68,7 +72,7 @@ pipeline {
                             sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
-                    stage('Run Cases') {
+                    stage("Run ${Cases}") {
                         steps {
                             echo "Do Test for ${CHOICE_NODE} - ${Cases}"
                         }
@@ -77,4 +81,13 @@ pipeline {
             }
         }
     }
+
+    post {
+        always {
+            script {
+                pullRequest.removeLabel('CI-${MACHINE^}-Building')
+                pullRequest.addLabel('CI-${MACHINE^}-Done')
+            }
+        }
+    }
 }
\ No newline at end of file
-- 
GitLab


From 5ed8d5d9a78a270c49b1a3c95803020b50b0bc7d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:44:48 +0000
Subject: [PATCH 136/447] tried to get Uppercase in MACHINE to work

---
 Jenkinsfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b5990db10..8082b3d5e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,11 +48,12 @@ pipeline {
                 stages {
                     stage('Build') {
                         steps {
-                           echo "Do Build for ${MACHINE^} - ${Cases}"
                            script {
-                            pullRequest.removeLabel('CI-${MACHINE^}-Ready')
-                            pullRequest.addLabel('CI-${MACHINE^}-Building')
+                            machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                            pullRequest.removeLabel("CI-${machine}-Ready")
+                            pullRequest.addLabel("CI-${machine}-Building")
                            }
+                           echo "Do Build for" ${machine} ${Cases}
                             cleanWs()
                             checkout scm
                             //sh 'sorc/build_all.sh -gu'
@@ -85,8 +86,8 @@ pipeline {
     post {
         always {
             script {
-                pullRequest.removeLabel('CI-${MACHINE^}-Building')
-                pullRequest.addLabel('CI-${MACHINE^}-Done')
+                pullRequest.removeLabel("CI-${machine}-Building")
+                pullRequest.addLabel("CI-${machine}-Done")
             }
         }
     }
-- 
GitLab


From 93ae7ba68731ea9260f00d2a21dab4d2a1356934 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:49:03 +0000
Subject: [PATCH 137/447] removed var in stage name

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8082b3d5e..5e3dca40a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -73,9 +73,9 @@ pipeline {
                             sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
-                    stage("Run ${Cases}") {
+                    stage("Run Cases") {
                         steps {
-                            echo "Do Test for ${CHOICE_NODE} - ${Cases}"
+                            echo "Do Test for ${machine} - ${Cases}"
                         }
                     }
                 }
-- 
GitLab


From 99c23d13367efe0db4337fd68bb3554a8da87a71 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 00:50:44 +0000
Subject: [PATCH 138/447] missed quote in echo

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5e3dca40a..34c32c555 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -53,7 +53,7 @@ pipeline {
                             pullRequest.removeLabel("CI-${machine}-Ready")
                             pullRequest.addLabel("CI-${machine}-Building")
                            }
-                           echo "Do Build for" ${machine} ${Cases}
+                           echo "Do Build for ${machine} ${Cases}"
                             cleanWs()
                             checkout scm
                             //sh 'sorc/build_all.sh -gu'
-- 
GitLab


From 51c1d757ea0f70b42b4e594e07c3d2793ca9fc48 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:03:55 +0000
Subject: [PATCH 139/447] removed wipe

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 34c32c555..af2b0747d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -54,7 +54,6 @@ pipeline {
                             pullRequest.addLabel("CI-${machine}-Building")
                            }
                            echo "Do Build for ${machine} ${Cases}"
-                            cleanWs()
                             checkout scm
                             //sh 'sorc/build_all.sh -gu'
                             sh 'sorc/link_workflow.sh'
-- 
GitLab


From 32cd5d7a0fec4852e59fedcac736209790a7c675 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:11:06 +0000
Subject: [PATCH 140/447] echo label

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index af2b0747d..85b9a7f45 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -14,6 +14,7 @@ pipeline {
             steps {
                 script {
                     for (label in pullRequest.labels) {
+                        echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
                              env.MACHINE='hera'
                         }  
-- 
GitLab


From 362b48c97b755190ce8f1ac92f712eca9728dc66 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:13:54 +0000
Subject: [PATCH 141/447] echo labels

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 85b9a7f45..3aa614a7d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,6 +13,7 @@ pipeline {
 
             steps {
                 script {
+                    echo pullRequest.labels
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
-- 
GitLab


From b79c13ec9a47a5f471814a3c33e40a0ad1a73f72 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:16:10 +0000
Subject: [PATCH 142/447] echo machine in post

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3aa614a7d..b8abb3a5a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -87,6 +87,8 @@ pipeline {
     post {
         always {
             script {
+                machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                echo "Do Post for ${machine}"
                 pullRequest.removeLabel("CI-${machine}-Building")
                 pullRequest.addLabel("CI-${machine}-Done")
             }
-- 
GitLab


From eba62614523a157c62f09760545175de4729b087 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:18:32 +0000
Subject: [PATCH 143/447] echo no op in post

---
 Jenkinsfile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b8abb3a5a..ea3643a6a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -87,10 +87,11 @@ pipeline {
     post {
         always {
             script {
-                machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
-                echo "Do Post for ${machine}"
-                pullRequest.removeLabel("CI-${machine}-Building")
-                pullRequest.addLabel("CI-${machine}-Done")
+                //machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                //echo "Do Post for ${machine}"
+                //pullRequest.removeLabel("CI-${machine}-Building")
+                //pullRequest.addLabel("CI-${machine}-Done")
+                echo "no-op"
             }
         }
     }
-- 
GitLab


From 4d3a87299d2436da8fb90b485bdbfe203687e8bb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:26:51 +0000
Subject: [PATCH 144/447] moved MACHINE out from env

---
 Jenkinsfile | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ea3643a6a..c1121db41 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -17,16 +17,16 @@ pipeline {
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
-                             env.MACHINE='hera'
+                             MACHINE='hera'
                         }  
                         else if ((label.matches("CI-Orion-Ready"))) {
-                            env.MACHINE='orion'
+                            MACHINE='orion'
                         }  
                         else if ((label.matches("CI-Hercules-Ready"))) {
-                            env.MACHINE='hercules'
+                            MACHINE='hercules'
                         }  
                         else { 
-                            env.MACHINE='none'
+                            MACHINE='none'
                         }
                      }
                 }
@@ -36,7 +36,7 @@ pipeline {
         stage( 'Build and Test' ) {
 
             when {
-                expression { env.MACHINE != 'none' }
+                expression { MACHINE != 'none' }
             }
 
             matrix {
@@ -87,10 +87,10 @@ pipeline {
     post {
         always {
             script {
-                //machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
-                //echo "Do Post for ${machine}"
-                //pullRequest.removeLabel("CI-${machine}-Building")
-                //pullRequest.addLabel("CI-${machine}-Done")
+                machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                echo "Do Post for ${machine}"
+                pullRequest.removeLabel("CI-${machine}-Building")
+                pullRequest.addLabel("CI-${machine}-Passed")
                 echo "no-op"
             }
         }
-- 
GitLab


From 52e20cd5123c61b3694028a969f5d9567f71bd72 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:29:03 +0000
Subject: [PATCH 145/447] deleted and echo

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c1121db41..342e31e9d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,7 +13,6 @@ pipeline {
 
             steps {
                 script {
-                    echo pullRequest.labels
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
-- 
GitLab


From 38713d6a17b5f0615a5b75167020f247c81858d1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:31:26 +0000
Subject: [PATCH 146/447] removed fail label in post

---
 Jenkinsfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 342e31e9d..bdeff84b8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -88,9 +88,8 @@ pipeline {
             script {
                 machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                 echo "Do Post for ${machine}"
-                pullRequest.removeLabel("CI-${machine}-Building")
-                pullRequest.addLabel("CI-${machine}-Passed")
-                echo "no-op"
+                //pullRequest.removeLabel("CI-${machine}-Building")
+                //pullRequest.addLabel("CI-${machine}-Passed")
             }
         }
     }
-- 
GitLab


From cb1240a5b32f80d6faf23de7a0ddfa82de323ed3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:37:33 +0000
Subject: [PATCH 147/447] missed and env on Macine

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bdeff84b8..3905bc8f7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
                     stage('Build') {
                         steps {
                            script {
-                            machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                             pullRequest.removeLabel("CI-${machine}-Ready")
                             pullRequest.addLabel("CI-${machine}-Building")
                            }
-- 
GitLab


From f8ebd722c5dd202932eb45623e4cc50e11dfbce2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:44:46 +0000
Subject: [PATCH 148/447] do not do else and make MACHINE none

---
 Jenkinsfile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3905bc8f7..0831a8233 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,6 +13,7 @@ pipeline {
 
             steps {
                 script {
+                    MACHINE = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
@@ -24,9 +25,6 @@ pipeline {
                         else if ((label.matches("CI-Hercules-Ready"))) {
                             MACHINE='hercules'
                         }  
-                        else { 
-                            MACHINE='none'
-                        }
                      }
                 }
             }
-- 
GitLab


From 943e5046d437d032cfe973ccf04c8fedb68b57b4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:50:34 +0000
Subject: [PATCH 149/447] add Machine ID

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0831a8233..2c2a42eaf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -55,6 +55,7 @@ pipeline {
                            echo "Do Build for ${machine} ${Cases}"
                             checkout scm
                             //sh 'sorc/build_all.sh -gu'
+                            script { env.MACHINE_ID = MACHINE } 
                             sh 'sorc/link_workflow.sh'
                         }
                     }
-- 
GitLab


From 62b10b28e4914fd7c7556b98084d3dcb7db92b0b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 01:55:08 +0000
Subject: [PATCH 150/447] can not remove twice

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2c2a42eaf..469deb74f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,8 +49,8 @@ pipeline {
                         steps {
                            script {
                             machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                            pullRequest.removeLabel("CI-${machine}-Ready")
-                            pullRequest.addLabel("CI-${machine}-Building")
+                            //pullRequest.removeLabel("CI-${machine}-Ready")
+                            //pullRequest.addLabel("CI-${machine}-Building")
                            }
                            echo "Do Build for ${machine} ${Cases}"
                             checkout scm
-- 
GitLab


From fd9f6b207536e41616099c2f50d7b9027450cb53 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:01:20 +0000
Subject: [PATCH 151/447] needed quotes around create Cases

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 469deb74f..2a1c6fbfc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -62,7 +62,7 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                             script { 
-                                env.case = ${Cases}
+                                env.case = "${Cases}"
                                 env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
                             }
                             sh '''
-- 
GitLab


From c952ef0d76e640659df4472b3821c4876cc22600 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:04:40 +0000
Subject: [PATCH 152/447] removed rm RUNTESTS duh

---
 Jenkinsfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2a1c6fbfc..864bf6153 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -65,10 +65,7 @@ pipeline {
                                 env.case = "${Cases}"
                                 env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
                             }
-                            sh '''
-                                rm -rf ${RUNTESTS}
-                                mkdir -p ${RUNTESTS}
-                                '''
+                            sh 'mkdir -p ${RUNTESTS}'
                             sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
-- 
GitLab


From 009f89708de3c3a388cabc5fb172e60920acafc4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:11:50 +0000
Subject: [PATCH 153/447] removed hybatmDA

---
 Jenkinsfile | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 864bf6153..a2108c2a6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
                 axes {
                     axis {
                         name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar', 'C96C48_hybatmDA'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar' //, 'C96C48_hybatmDA'
                     }
                 }
                 stages {
@@ -80,12 +80,18 @@ pipeline {
     }
 
     post {
-        always {
+        success {
             script {
                 machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                 echo "Do Post for ${machine}"
-                //pullRequest.removeLabel("CI-${machine}-Building")
-                //pullRequest.addLabel("CI-${machine}-Passed")
+                pullRequest.removeLabel("CI-${machine}-Building")
+                pullRequest.addLabel("CI-${machine}-Passed")
+            }
+        failure {
+            script {
+                machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                pullRequest.removeLabel('CI-${machine}-Running')
+                pullRequest.addLabel('CI-${machine}-Failed')  
             }
         }
     }
-- 
GitLab


From 2688bf970b28038c2fbf8d2c8c8935000f40f6eb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:13:21 +0000
Subject: [PATCH 154/447] label change

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a2108c2a6..92553334f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,8 +49,8 @@ pipeline {
                         steps {
                            script {
                             machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                            //pullRequest.removeLabel("CI-${machine}-Ready")
-                            //pullRequest.addLabel("CI-${machine}-Building")
+                            pullRequest.removeLabel("CI-${machine}-Ready")
+                            pullRequest.addLabel("CI-${machine}-Building")
                            }
                            echo "Do Build for ${machine} ${Cases}"
                             checkout scm
-- 
GitLab


From 370d328be594879500cf4657bfc0ce01aad08e25 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:18:46 +0000
Subject: [PATCH 155/447] braclets

---
 Jenkinsfile | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 92553334f..3f746fc93 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
                 axes {
                     axis {
                         name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar' //, 'C96C48_hybatmDA'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar'
                     }
                 }
                 stages {
@@ -78,21 +78,22 @@ pipeline {
             }
         }
     }
-
-    post {
-        success {
-            script {
-                machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                echo "Do Post for ${machine}"
-                pullRequest.removeLabel("CI-${machine}-Building")
-                pullRequest.addLabel("CI-${machine}-Passed")
-            }
-        failure {
-            script {
-                machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                pullRequest.removeLabel('CI-${machine}-Running')
-                pullRequest.addLabel('CI-${machine}-Failed')  
-            }
+}
+    
+post {
+    success {
+        script {
+            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+            echo "Do Post for ${machine}"
+            pullRequest.removeLabel("CI-${machine}-Building")
+            pullRequest.addLabel("CI-${machine}-Passed")
+        }
+    }
+    failure {
+        script {
+            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+            pullRequest.removeLabel('CI-${machine}-Running')
+            pullRequest.addLabel('CI-${machine}-Failed')  
         }
     }
 }
\ No newline at end of file
-- 
GitLab


From 46274bc910d5818a3090a69a2e739ee7a252d851 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Jan 2024 02:41:50 +0000
Subject: [PATCH 156/447] added yes to create experemnt

---
 Jenkinsfile                  | 2 +-
 ci/scripts/utils/ci_utils.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3f746fc93..c35947d67 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -54,7 +54,7 @@ pipeline {
                            }
                            echo "Do Build for ${machine} ${Cases}"
                             checkout scm
-                            //sh 'sorc/build_all.sh -gu'
+                            sh 'sorc/build_all.sh -gu'
                             script { env.MACHINE_ID = MACHINE } 
                             sh 'sorc/link_workflow.sh'
                         }
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index d46688bd6..18ed03707 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -61,6 +61,6 @@ function create_experiment () {
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
   export pslot=${case}_${pr_sha}
-  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  yes | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
\ No newline at end of file
-- 
GitLab


From 0583ee12928db1595ca6cc11c27fbd0ef41f1dba Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 18:26:49 +0000
Subject: [PATCH 157/447] moved build step outside of Matrix block so that all
 cases run against a single build

---
 Jenkinsfile | 75 ++++++++++++-----------------------------------------
 1 file changed, 16 insertions(+), 59 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c35947d67..6123c79a3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,43 +1,32 @@
 pipeline {
-    agent none
-
-    options {
-        disableConcurrentBuilds()
-        overrideIndexTriggers(false)
-        skipDefaultCheckout(true)
-    }
+    agent any
 
     stages {
-
-        stage( 'Get Machine' ) {
-
+        stage('Build') {
+            when {
+                expression { env.MACHINE != 'none' }
+            }
             steps {
                 script {
-                    MACHINE = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches("CI-Hera-Ready"))) {
-                             MACHINE='hera'
-                        }  
-                        else if ((label.matches("CI-Orion-Ready"))) {
-                            MACHINE='orion'
-                        }  
-                        else if ((label.matches("CI-Hercules-Ready"))) {
-                            MACHINE='hercules'
-                        }  
-                     }
+                    machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                    pullRequest.removeLabel("CI-${machine}-Ready")
+                    pullRequest.addLabel("CI-${machine}-Building")
                 }
+                echo "Do Build for ${machine}"
+                checkout scm
+                sh 'sorc/build_all.sh -gu'
+                script { env.MACHINE_ID = env.MACHINE } 
+                sh 'sorc/link_workflow.sh'
             }
         }
 
-        stage( 'Build and Test' ) {
-
+        stage('Build and Test') {
             when {
-                expression { MACHINE != 'none' }
+                expression { env.MACHINE != 'none' }
             }
 
             matrix {
-                agent { label "${MACHINE}-emc" }
+                agent { label "${env.MACHINE}-emc" }
                 axes {
                     axis {
                         name 'Cases'
@@ -45,20 +34,6 @@ pipeline {
                     }
                 }
                 stages {
-                    stage('Build') {
-                        steps {
-                           script {
-                            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                            pullRequest.removeLabel("CI-${machine}-Ready")
-                            pullRequest.addLabel("CI-${machine}-Building")
-                           }
-                           echo "Do Build for ${machine} ${Cases}"
-                            checkout scm
-                            sh 'sorc/build_all.sh -gu'
-                            script { env.MACHINE_ID = MACHINE } 
-                            sh 'sorc/link_workflow.sh'
-                        }
-                    }
                     stage('Create Experiment') {
                         steps {
                             script { 
@@ -78,22 +53,4 @@ pipeline {
             }
         }
     }
-}
-    
-post {
-    success {
-        script {
-            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-            echo "Do Post for ${machine}"
-            pullRequest.removeLabel("CI-${machine}-Building")
-            pullRequest.addLabel("CI-${machine}-Passed")
-        }
-    }
-    failure {
-        script {
-            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-            pullRequest.removeLabel('CI-${machine}-Running')
-            pullRequest.addLabel('CI-${machine}-Failed')  
-        }
-    }
 }
\ No newline at end of file
-- 
GitLab


From 55f7103e5bb6e6f7f9b748d74d1f4af5b549d79a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 18:44:57 +0000
Subject: [PATCH 158/447] Add machine selection based on labels in Jenkinsfile

---
 Jenkinsfile          | 28 ++++++++++++-
 Jenkinsfile_arch_opt | 99 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)
 create mode 100644 Jenkinsfile_arch_opt

diff --git a/Jenkinsfile b/Jenkinsfile
index 6123c79a3..9695e363e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,31 @@
 pipeline {
-    agent any
+
+    options {
+        disableConcurrentBuilds()
+        overrideIndexTriggers(false)
+        skipDefaultCheckout(true)
+    }
+
+    stage( 'Get Machine' ) {
+        agent { label 'master' }
+        steps {
+            script {
+                MACHINE = 'none'
+                for (label in pullRequest.labels) {
+                    echo "Label: ${label}"
+                    if ((label.matches("CI-Hera-Ready"))) {
+                        MACHINE='hera'
+                    }  
+                    else if ((label.matches("CI-Orion-Ready"))) {
+                        MACHINE='orion'
+                    }  
+                    else if ((label.matches("CI-Hercules-Ready"))) {
+                            MACHINE='hercules'
+                    }  
+                }
+            }
+        }
+    }
 
     stages {
         stage('Build') {
diff --git a/Jenkinsfile_arch_opt b/Jenkinsfile_arch_opt
new file mode 100644
index 000000000..c35947d67
--- /dev/null
+++ b/Jenkinsfile_arch_opt
@@ -0,0 +1,99 @@
+pipeline {
+    agent none
+
+    options {
+        disableConcurrentBuilds()
+        overrideIndexTriggers(false)
+        skipDefaultCheckout(true)
+    }
+
+    stages {
+
+        stage( 'Get Machine' ) {
+
+            steps {
+                script {
+                    MACHINE = 'none'
+                    for (label in pullRequest.labels) {
+                        echo "Label: ${label}"
+                        if ((label.matches("CI-Hera-Ready"))) {
+                             MACHINE='hera'
+                        }  
+                        else if ((label.matches("CI-Orion-Ready"))) {
+                            MACHINE='orion'
+                        }  
+                        else if ((label.matches("CI-Hercules-Ready"))) {
+                            MACHINE='hercules'
+                        }  
+                     }
+                }
+            }
+        }
+
+        stage( 'Build and Test' ) {
+
+            when {
+                expression { MACHINE != 'none' }
+            }
+
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                axes {
+                    axis {
+                        name 'Cases'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar'
+                    }
+                }
+                stages {
+                    stage('Build') {
+                        steps {
+                           script {
+                            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                            pullRequest.removeLabel("CI-${machine}-Ready")
+                            pullRequest.addLabel("CI-${machine}-Building")
+                           }
+                           echo "Do Build for ${machine} ${Cases}"
+                            checkout scm
+                            sh 'sorc/build_all.sh -gu'
+                            script { env.MACHINE_ID = MACHINE } 
+                            sh 'sorc/link_workflow.sh'
+                        }
+                    }
+                    stage('Create Experiment') {
+                        steps {
+                            script { 
+                                env.case = "${Cases}"
+                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                            }
+                            sh 'mkdir -p ${RUNTESTS}'
+                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                        }
+                    }
+                    stage("Run Cases") {
+                        steps {
+                            echo "Do Test for ${machine} - ${Cases}"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+    
+post {
+    success {
+        script {
+            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+            echo "Do Post for ${machine}"
+            pullRequest.removeLabel("CI-${machine}-Building")
+            pullRequest.addLabel("CI-${machine}-Passed")
+        }
+    }
+    failure {
+        script {
+            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+            pullRequest.removeLabel('CI-${machine}-Running')
+            pullRequest.addLabel('CI-${machine}-Failed')  
+        }
+    }
+}
\ No newline at end of file
-- 
GitLab


From 502ee8e603fb1e6677d094be48993341d5ba30a5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 18:48:37 +0000
Subject: [PATCH 159/447] Add agent label for build stage and global master

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9695e363e..ce6ce7f05 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,6 @@
 pipeline {
 
+    agent { label 'master' }
     options {
         disableConcurrentBuilds()
         overrideIndexTriggers(false)
@@ -29,6 +30,7 @@ pipeline {
 
     stages {
         stage('Build') {
+            agent { label "${env.MACHINE}-emc" }
             when {
                 expression { env.MACHINE != 'none' }
             }
-- 
GitLab


From 404259f39c45acb813aee0d7ccc5bbb812411483 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 18:57:17 +0000
Subject: [PATCH 160/447] moved main stages block to top

---
 Jenkinsfile | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ce6ce7f05..16462cbea 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,28 +7,27 @@ pipeline {
         skipDefaultCheckout(true)
     }
 
-    stage( 'Get Machine' ) {
-        agent { label 'master' }
-        steps {
-            script {
-                MACHINE = 'none'
-                for (label in pullRequest.labels) {
-                    echo "Label: ${label}"
-                    if ((label.matches("CI-Hera-Ready"))) {
-                        MACHINE='hera'
-                    }  
-                    else if ((label.matches("CI-Orion-Ready"))) {
-                        MACHINE='orion'
-                    }  
-                    else if ((label.matches("CI-Hercules-Ready"))) {
-                            MACHINE='hercules'
-                    }  
+    stages {
+
+        stage('Get Machine') {
+            agent { label 'master' }
+            steps {
+                script {
+                    MACHINE = 'none'
+                    for (label in pullRequest.labels) {
+                        echo "Label: ${label}"
+                        if ((label.matches("CI-Hera-Ready"))) {
+                            MACHINE = 'hera'
+                        } else if ((label.matches("CI-Orion-Ready"))) {
+                            MACHINE = 'orion'
+                        } else if ((label.matches("CI-Hercules-Ready"))) {
+                            MACHINE = 'hercules'
+                        }
+                    }
                 }
             }
         }
-    }
 
-    stages {
         stage('Build') {
             agent { label "${env.MACHINE}-emc" }
             when {
@@ -43,7 +42,7 @@ pipeline {
                 echo "Do Build for ${machine}"
                 checkout scm
                 sh 'sorc/build_all.sh -gu'
-                script { env.MACHINE_ID = env.MACHINE } 
+                script { env.MACHINE_ID = env.MACHINE }
                 sh 'sorc/link_workflow.sh'
             }
         }
@@ -58,13 +57,13 @@ pipeline {
                 axes {
                     axis {
                         name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar'
+                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar'
                     }
                 }
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            script { 
+                            script {
                                 env.case = "${Cases}"
                                 env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
                             }
@@ -80,5 +79,6 @@ pipeline {
                 }
             }
         }
+
     }
 }
\ No newline at end of file
-- 
GitLab


From a40a55a5f21580c8d82d862b1927daa91613b3c4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 19:11:43 +0000
Subject: [PATCH 161/447] none agent on top and buit-in before assinged remote
 agent

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 16462cbea..2e4824c6c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,6 @@
 pipeline {
 
-    agent { label 'master' }
+    agent none
     options {
         disableConcurrentBuilds()
         overrideIndexTriggers(false)
@@ -10,7 +10,7 @@ pipeline {
     stages {
 
         stage('Get Machine') {
-            agent { label 'master' }
+            agent { label 'build-in' }
             steps {
                 script {
                     MACHINE = 'none'
-- 
GitLab


From c0b8c4fcade67c33a9398ac9c11ad5a0b212f79b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 19:14:25 +0000
Subject: [PATCH 162/447] build in to buit in type fix

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2e4824c6c..878de6789 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -10,7 +10,7 @@ pipeline {
     stages {
 
         stage('Get Machine') {
-            agent { label 'build-in' }
+            agent { label 'build-it' }
             steps {
                 script {
                     MACHINE = 'none'
-- 
GitLab


From 34be579bcda0c50d092bdc7a4a09cbf353b9c0ce Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 19:16:30 +0000
Subject: [PATCH 163/447] still typed buid-in wrong git add Jenkinsfile ! grrr

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 878de6789..2e4824c6c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -10,7 +10,7 @@ pipeline {
     stages {
 
         stage('Get Machine') {
-            agent { label 'build-it' }
+            agent { label 'build-in' }
             steps {
                 script {
                     MACHINE = 'none'
-- 
GitLab


From 3ef2866ad59b9b8ec0f4d5993c68f912b9b713ad Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 19:18:09 +0000
Subject: [PATCH 164/447] OMG I typed built-in yet one more time

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2e4824c6c..1f1ef218f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -10,7 +10,7 @@ pipeline {
     stages {
 
         stage('Get Machine') {
-            agent { label 'build-in' }
+            agent { label 'built-in' }
             steps {
                 script {
                     MACHINE = 'none'
-- 
GitLab


From e22f33404c6c85c0dc75c6b0668b78104685513a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 19:24:36 +0000
Subject: [PATCH 165/447] moved MACHINE out of env scpope and make global to
 Jenkins

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1f1ef218f..082a41f42 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -29,27 +29,27 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "${env.MACHINE}-emc" }
+            agent { label "${MACHINE}-emc" }
             when {
-                expression { env.MACHINE != 'none' }
+                expression { MACHINE != 'none' }
             }
             steps {
                 script {
-                    machine = env.MACHINE[0].toUpperCase() + env.MACHINE.substring(1)
+                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                 }
                 echo "Do Build for ${machine}"
                 checkout scm
+                script { env.MACHINE_ID = MACHINE }
                 sh 'sorc/build_all.sh -gu'
-                script { env.MACHINE_ID = env.MACHINE }
                 sh 'sorc/link_workflow.sh'
             }
         }
 
         stage('Build and Test') {
             when {
-                expression { env.MACHINE != 'none' }
+                expression { MACHINE != 'none' }
             }
 
             matrix {
-- 
GitLab


From 632f8f7be943a720a9081df77349745ac11f08d5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 20:24:36 +0000
Subject: [PATCH 166/447] left env on last global MACHINE var in creating the
 experments

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 082a41f42..2d0774743 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -53,7 +53,7 @@ pipeline {
             }
 
             matrix {
-                agent { label "${env.MACHINE}-emc" }
+                agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name 'Cases'
-- 
GitLab


From 89e4dc756407a27f098f03c8efd5ecb93385d751 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 20:25:58 +0000
Subject: [PATCH 167/447] skip actual buiding for testing

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2d0774743..fe4c31ff7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                 echo "Do Build for ${machine}"
                 checkout scm
                 script { env.MACHINE_ID = MACHINE }
-                sh 'sorc/build_all.sh -gu'
+                //sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
             }
         }
-- 
GitLab


From b64f45759af7b0fff4e7a5e1fb8b80490ac14edd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 20:45:10 +0000
Subject: [PATCH 168/447] skipDefaultCheckout may had wrong syntax, also
 removed yes pip to create experement

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe4c31ff7..a63c3e4c1 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,9 +2,9 @@ pipeline {
 
     agent none
     options {
-        disableConcurrentBuilds()
-        overrideIndexTriggers(false)
-        skipDefaultCheckout(true)
+        disableConcurrentBuilds(abortProcess: true)
+        skipDefaultCheckout()
+        buildDicarder(logRotator(numToKeepStr: '2'))
     }
 
     stages {
@@ -42,7 +42,7 @@ pipeline {
                 echo "Do Build for ${machine}"
                 checkout scm
                 script { env.MACHINE_ID = MACHINE }
-                //sh 'sorc/build_all.sh -gu'
+                // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
             }
         }
-- 
GitLab


From 82b69ff48fa7e69641cbc5e082a87a6bada2de70 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 20:47:34 +0000
Subject: [PATCH 169/447] updated to abortPrevious

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a63c3e4c1..f29634365 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,7 +2,7 @@ pipeline {
 
     agent none
     options {
-        disableConcurrentBuilds(abortProcess: true)
+        disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout()
         buildDicarder(logRotator(numToKeepStr: '2'))
     }
-- 
GitLab


From a7826e6c90fa91c15392a79ee3b4e011af326196 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 20:50:03 +0000
Subject: [PATCH 170/447] spelled buildDiscarder wrong and fixed

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f29634365..6a5df28e2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,7 +4,7 @@ pipeline {
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout()
-        buildDicarder(logRotator(numToKeepStr: '2'))
+        buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
     stages {
-- 
GitLab


From 599ad702f6e20c39ecf9168dd4e2b1c627a63af3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 21:05:34 +0000
Subject: [PATCH 171/447] remove the workspace path and leave it to the common
 sc area

---
 Jenkinsfile                  | 4 ++--
 ci/scripts/utils/ci_utils.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6a5df28e2..e2ca57dd3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -65,10 +65,10 @@ pipeline {
                         steps {
                             script {
                                 env.case = "${Cases}"
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
+                                env.RUNTESTS = "RUNTESTS"
                             }
                             sh 'mkdir -p ${RUNTESTS}'
-                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                            sh 'ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
                         }
                     }
                     stage("Run Cases") {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 18ed03707..d46688bd6 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -61,6 +61,6 @@ function create_experiment () {
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
   export pslot=${case}_${pr_sha}
-  yes | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
\ No newline at end of file
-- 
GitLab


From c6a9f84d99d271dd1267b9546638f92fbc75a58a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 21:18:37 +0000
Subject: [PATCH 172/447] trying to force HOMEgfs from scm WORKSPACE

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e2ca57dd3..c99307937 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,6 +44,7 @@ pipeline {
                 script { env.MACHINE_ID = MACHINE }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
+                HOMEgfs="${WORKSPACE}"
             }
         }
 
@@ -66,9 +67,10 @@ pipeline {
                             script {
                                 env.case = "${Cases}"
                                 env.RUNTESTS = "RUNTESTS"
+                                env.HOMEgfs = "${HOMEgfs}"
                             }
-                            sh 'mkdir -p ${RUNTESTS}'
-                            sh 'ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
+                            sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}'
+                            sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml'
                         }
                     }
                     stage("Run Cases") {
-- 
GitLab


From 264363ad2025f3148e1eee9623c9345fcc84478c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 21:22:40 +0000
Subject: [PATCH 173/447] moved assignment of Jenkins HOMEgfs into sript block

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c99307937..35cb41a94 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,10 +41,12 @@ pipeline {
                 }
                 echo "Do Build for ${machine}"
                 checkout scm
-                script { env.MACHINE_ID = MACHINE }
+                script {
+                    env.MACHINE_ID = MACHINE
+                    HOMEgfs="${WORKSPACE}"
+                }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
-                HOMEgfs="${WORKSPACE}"
             }
         }
 
-- 
GitLab


From d07c1abd4f9de6258d3b5f743a43e95fce7d449b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 21:40:12 +0000
Subject: [PATCH 174/447] moved common mkdir for RUNCASES dir and hope matrix
 is resetting env.case

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 35cb41a94..28d35af1e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,11 +42,13 @@ pipeline {
                 echo "Do Build for ${machine}"
                 checkout scm
                 script {
+                    HOMEgfs = "${WORKSPACE}"
+                    env.HOMEgfs = "${HOMEgfs}"
                     env.MACHINE_ID = MACHINE
-                    HOMEgfs="${WORKSPACE}"
                 }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
+                sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}'
             }
         }
 
@@ -71,7 +73,6 @@ pipeline {
                                 env.RUNTESTS = "RUNTESTS"
                                 env.HOMEgfs = "${HOMEgfs}"
                             }
-                            sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}'
                             sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml'
                         }
                     }
-- 
GitLab


From 58192282b288a05ddfa4b14f5c2b8d11ee9778ac Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 21:57:08 +0000
Subject: [PATCH 175/447] degugging Case control variable in shell

---
 Jenkinsfile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 28d35af1e..7cc418a29 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,7 +48,7 @@ pipeline {
                 }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
-                sh 'mkdir -p ${HOMEgfs}/${RUNTESTS}'
+                sh 'mkdir -p ${HOMEgfs}/RUNTESTS'
             }
         }
 
@@ -61,8 +61,8 @@ pipeline {
                 agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
-                        name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar'
+                        name "Cases"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -70,9 +70,10 @@ pipeline {
                         steps {
                             script {
                                 env.case = "${Cases}"
-                                env.RUNTESTS = "RUNTESTS"
                                 env.HOMEgfs = "${HOMEgfs}"
                             }
+                            echo "Cases: ${Cases}"
+                            sh 'echo "Create Experiment for ${case} and ${Cases}"'
                             sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml'
                         }
                     }
-- 
GitLab


From 74eae6d461b82edf970b08a10c0906ba41e20d15 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:00:17 +0000
Subject: [PATCH 176/447] Cases does work in shell

---
 Jenkinsfile | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7cc418a29..d732b821e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,13 +68,8 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            script {
-                                env.case = "${Cases}"
-                                env.HOMEgfs = "${HOMEgfs}"
-                            }
-                            echo "Cases: ${Cases}"
-                            sh 'echo "Create Experiment for ${case} and ${Cases}"'
-                            sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${case}.yaml'
+                            sh 'echo "Create Experiment for ${Cases}"'
+                            sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
                     stage("Run Cases") {
-- 
GitLab


From 678e01817e6b2a11774d28be02f0d44538c8edb0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:11:26 +0000
Subject: [PATCH 177/447] issue with HOMEgfs in env

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d732b821e..8133cf294 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,7 +68,8 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            sh 'echo "Create Experiment for ${Cases}"'
+                            script { env.HOMEgfs = "${HOMEgfs}" }
+                            echo "Cases: ${Cases}"
                             sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
-- 
GitLab


From a9de618b0857713c5fca684a8e5bd0c3839aa121 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:15:27 +0000
Subject: [PATCH 178/447] move HOMEgfs to HOME

---
 Jenkinsfile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8133cf294..d0f7f1530 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,8 +42,7 @@ pipeline {
                 echo "Do Build for ${machine}"
                 checkout scm
                 script {
-                    HOMEgfs = "${WORKSPACE}"
-                    env.HOMEgfs = "${HOMEgfs}"
+                    HOME = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                 }
                 // sh 'sorc/build_all.sh -gu'
@@ -68,9 +67,8 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            script { env.HOMEgfs = "${HOMEgfs}" }
                             echo "Cases: ${Cases}"
-                            sh '${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Cases}.yaml'
+                            sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
                     stage("Run Cases") {
-- 
GitLab


From d5304ed283b31b91b64819506cc0678085a5ae0a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:17:40 +0000
Subject: [PATCH 179/447] left an extra HOMEgfs in place for mkdir

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d0f7f1530..7a6293009 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
                 }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
-                sh 'mkdir -p ${HOMEgfs}/RUNTESTS'
+                sh 'mkdir -p ${HOME}/RUNTESTS'
             }
         }
 
-- 
GitLab


From 66e507265e21aa40b5be3be799e25bb67a1e0755 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:21:27 +0000
Subject: [PATCH 180/447] seemed to loose HOME

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7a6293009..3824c3471 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -67,7 +67,8 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            echo "Cases: ${Cases}"
+                            script { env.HOME = "$HOME"}
+                            echo "Cases: ${Cases} ${HOME}"
                             sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
-- 
GitLab


From bbef554e4e34a029f34cf4cf11210d2fe6782adc Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:42:27 +0000
Subject: [PATCH 181/447] need RUNTESTS in shell env

---
 Jenkinsfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3824c3471..1368d8248 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,11 +47,11 @@ pipeline {
                 }
                 // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
-                sh 'mkdir -p ${HOME}/RUNTESTS'
+                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
             }
         }
 
-        stage('Build and Test') {
+        stage('Run Tests') {
             when {
                 expression { MACHINE != 'none' }
             }
@@ -67,12 +67,14 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            script { env.HOME = "$HOME"}
+                            script {
+                                 env.HOME = "$HOME"
+                                 env.RUNTESTS = "${HOME}/RUNTESTS"
                             echo "Cases: ${Cases} ${HOME}"
                             sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
-                    stage("Run Cases") {
+                    stage("Run Experiment") {
                         steps {
                             echo "Do Test for ${machine} - ${Cases}"
                         }
-- 
GitLab


From fbcec52c4d82b9296fb91f6a2385193013997600 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 9 Jan 2024 22:44:43 +0000
Subject: [PATCH 182/447] missed a bracket on setting local env

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1368d8248..1097d3686 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -70,6 +70,7 @@ pipeline {
                             script {
                                  env.HOME = "$HOME"
                                  env.RUNTESTS = "${HOME}/RUNTESTS"
+                            }
                             echo "Cases: ${Cases} ${HOME}"
                             sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
-- 
GitLab


From 01267ed873616f508d4d991f746617836ecf631c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 20:15:22 +0000
Subject: [PATCH 183/447] uncommented out the actual build

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1097d3686..5d8503943 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@ pipeline {
                     HOME = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                 }
-                // sh 'sorc/build_all.sh -gu'
+                sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
                 sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
             }
-- 
GitLab


From e2fe2b283fc82d002d55b764ea90c54107743242 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 22:27:14 +0000
Subject: [PATCH 184/447] added Jeinkins code for running experiments within
 the matrix block

---
 Jenkinsfile                  | 12 ++++++++----
 ci/scripts/utils/ci_utils.sh | 24 ++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5d8503943..c9fb15c60 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -75,14 +75,18 @@ pipeline {
                             sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
-                    stage("Run Experiment") {
-                        steps {
-                            echo "Do Test for ${machine} - ${Cases}"
+
+                    stage('Run Experiments') {
+                        script {
+                            pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                            sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
+                            pullRequest.comment("SUCCESS running experiments: ${Case} on Orion")
                         }
                     }
                 }
             }
-        }
+        }  
+
 
     }
 }
\ No newline at end of file
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index d46688bd6..06090f410 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -40,8 +40,8 @@ function get_pslot_list () {
     local RUNTESTS="${1}"
   
     #############################################################
-    # loop over every yaml file in the PR's ci/cases
-    # and create an run directory for each one for this PR loop
+    # loop over expdir directories in RUNTESTS
+    # and create list of the directory names (pslot) with the hash tag
     #############################################################
     for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do
       pslot=$(basename "${pslot_dir}") || true
@@ -50,6 +50,26 @@ function get_pslot_list () {
 
 }
 
+function get_pslot () {
+
+    local RUNTESTS="${1}"
+    local case="${2}"
+  
+    #############################################################
+    # loop over expdir directories in RUNTESTS
+    # and return the name of the pslot with its tag that matches the case
+    #############################################################
+    for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do
+      pslot=$(basename "${pslot_dir}")
+      check_case = $(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true
+      if [[ "${check_case}" == "${case}" ]]; then
+        echo "${pslot}"
+        break
+      fi
+    done
+
+}
+
 function create_experiment () {
 
   local yaml_config="${1}"
-- 
GitLab


From 26d8c32a721b1500a8b9779e8768ef3042cec487 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 22:34:39 +0000
Subject: [PATCH 185/447] type in bash assign statment for check_case

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 06090f410..5716654c1 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -61,7 +61,7 @@ function get_pslot () {
     #############################################################
     for pslot_dir in "${RUNTESTS}/EXPDIR/"*; do
       pslot=$(basename "${pslot_dir}")
-      check_case = $(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true
+      check_case=$(echo "${pslot}" | rev | cut -d"_" -f2- | rev) || true
       if [[ "${check_case}" == "${case}" ]]; then
         echo "${pslot}"
         break
-- 
GitLab


From 90be03e214372e4b151cb91fcd8d81f358d3ace5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 22:46:00 +0000
Subject: [PATCH 186/447] added lables for Running and Pass/Fail

---
 Jenkinsfile | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c9fb15c60..f7188e577 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -79,14 +79,30 @@ pipeline {
                     stage('Run Experiments') {
                         script {
                             pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                            pullRequest.removeLabel('CI-${machine}-Building')
+                            pullRequest.addLabel('CI-${machine}-Running')
                             sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                             pullRequest.comment("SUCCESS running experiments: ${Case} on Orion")
                         }
                     }
                 }
             }
-        }  
+        }
 
+        post {
+            success {
+                script {
+                    pullRequest.removeLabel('CI-${machine}-Running')
+                    pullRequest.addLabel('CI-${machine}-Passed')  
+                }
+            }
+            failure {
+                script {
+                    pullRequest.removeLabel('CI-${machine}-Running')
+                    pullRequest.addLabel('CI-${machine}-Failed')  
+                }
+            }
+        }   
 
     }
 }
\ No newline at end of file
-- 
GitLab


From cd11a82d53776428c50cc8ecadf3adbe7bc9ac09 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 22:49:55 +0000
Subject: [PATCH 187/447] missed a bracket placment for stages block to
 seperate from post

---
 Jenkinsfile | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f7188e577..60d075f74 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -88,21 +88,21 @@ pipeline {
                 }
             }
         }
+    }
 
-        post {
-            success {
-                script {
-                    pullRequest.removeLabel('CI-${machine}-Running')
-                    pullRequest.addLabel('CI-${machine}-Passed')  
-                }
+    post {
+        success {
+            script {
+                pullRequest.removeLabel('CI-${machine}-Running')
+                pullRequest.addLabel('CI-${machine}-Passed')  
             }
-            failure {
-                script {
-                    pullRequest.removeLabel('CI-${machine}-Running')
-                    pullRequest.addLabel('CI-${machine}-Failed')  
-                }
+        }
+        failure {
+            script {
+                pullRequest.removeLabel('CI-${machine}-Running')
+                pullRequest.addLabel('CI-${machine}-Failed')  
             }
-        }   
-
+        }
     }
-}
\ No newline at end of file
+
+}
-- 
GitLab


From 2ef90391e329b66eb326a86646da058fc48ccba1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Jan 2024 22:54:53 +0000
Subject: [PATCH 188/447] needed steps in Run stage

---
 Jenkinsfile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 60d075f74..2e44bd305 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -77,12 +77,14 @@ pipeline {
                     }
 
                     stage('Run Experiments') {
-                        script {
-                            pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                            pullRequest.removeLabel('CI-${machine}-Building')
-                            pullRequest.addLabel('CI-${machine}-Running')
+                        steps {
+                            script {
+                                pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                pullRequest.removeLabel('CI-${machine}-Building')
+                                pullRequest.addLabel('CI-${machine}-Running')
+                            }
                             sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
-                            pullRequest.comment("SUCCESS running experiments: ${Case} on Orion")
+                            script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") }
                         }
                     }
                 }
-- 
GitLab


From 71e3c0060628d57beecaf53a85a60ee4f0b3537c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 00:12:15 +0000
Subject: [PATCH 189/447] removed labels to debug

---
 Jenkinsfile | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2e44bd305..7fd07e261 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -80,8 +80,8 @@ pipeline {
                         steps {
                             script {
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                pullRequest.removeLabel('CI-${machine}-Building')
-                                pullRequest.addLabel('CI-${machine}-Running')
+                                //pullRequest.removeLabel('CI-${machine}-Building')
+                                //pullRequest.addLabel('CI-${machine}-Running')
                             }
                             sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                             script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") }
@@ -95,14 +95,16 @@ pipeline {
     post {
         success {
             script {
-                pullRequest.removeLabel('CI-${machine}-Running')
-                pullRequest.addLabel('CI-${machine}-Passed')  
+                echo "SUCCESS"
+                //pullRequest.removeLabel('CI-${machine}-Running')
+                //pullRequest.addLabel('CI-${machine}-Passed')  
             }
         }
         failure {
             script {
-                pullRequest.removeLabel('CI-${machine}-Running')
-                pullRequest.addLabel('CI-${machine}-Failed')  
+                echo "FAILURE"
+                //pullRequest.removeLabel('CI-${machine}-Running')
+                //pullRequest.addLabel('CI-${machine}-Failed')  
             }
         }
     }
-- 
GitLab


From 2cf68f577cdd9c02d97dae2fea92690c3dd1fa22 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 00:59:27 +0000
Subject: [PATCH 190/447] Need to rethink Labeling because of the parallel
 nature of the matrix block

---
 Jenkinsfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7fd07e261..7f6dcd208 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -36,8 +36,8 @@ pipeline {
             steps {
                 script {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                    pullRequest.removeLabel("CI-${machine}-Ready")
-                    pullRequest.addLabel("CI-${machine}-Building")
+                    //pullRequest.removeLabel("CI-${machine}-Ready")
+                    //pullRequest.addLabel("CI-${machine}-Building")
                 }
                 echo "Do Build for ${machine}"
                 checkout scm
@@ -83,8 +83,9 @@ pipeline {
                                 //pullRequest.removeLabel('CI-${machine}-Building')
                                 //pullRequest.addLabel('CI-${machine}-Running')
                             }
+                            script { pullRequest.comment("Running experiments: ${Case} on ${machine}") }
                             sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
-                            script { pullRequest.comment("SUCCESS running experiments: ${Case} on Orion") }
+                            script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") }
                         }
                     }
                 }
-- 
GitLab


From c3997537f1236049efa3569b6ba915a5d14c1f4b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 00:59:53 +0000
Subject: [PATCH 191/447] Need to rethink Labeling because of the parallel
 nature of the matrix block

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7f6dcd208..0aea22e5a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@ pipeline {
                     HOME = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                 }
-                sh 'sorc/build_all.sh -gu'
+                //sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
                 sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
             }
-- 
GitLab


From fcf0e2816d061506dd4b94bb270fb16ec02cfaab Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 01:08:14 +0000
Subject: [PATCH 192/447] Trying to figure out Run Experiments block is
 complaining about not having steps, moved agent

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0aea22e5a..5481a951d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -52,12 +52,12 @@ pipeline {
         }
 
         stage('Run Tests') {
+            agent { label "${MACHINE}-emc" }
             when {
                 expression { MACHINE != 'none' }
             }
-
             matrix {
-                agent { label "${MACHINE}-emc" }
+                //agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name "Cases"
@@ -82,8 +82,8 @@ pipeline {
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 //pullRequest.removeLabel('CI-${machine}-Building')
                                 //pullRequest.addLabel('CI-${machine}-Running')
+                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             }
-                            script { pullRequest.comment("Running experiments: ${Case} on ${machine}") }
                             sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                             script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") }
                         }
-- 
GitLab


From c7b69daf53203849381d3522e3963442e160f118 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 01:11:29 +0000
Subject: [PATCH 193/447] moved agent back and coslidated script lines still
 get weird message about no steps in stage when they are clearly there

---
 Jenkinsfile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5481a951d..9aa494c35 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -52,12 +52,11 @@ pipeline {
         }
 
         stage('Run Tests') {
-            agent { label "${MACHINE}-emc" }
             when {
                 expression { MACHINE != 'none' }
             }
             matrix {
-                //agent { label "${MACHINE}-emc" }
+                agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name "Cases"
@@ -75,7 +74,6 @@ pipeline {
                             sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
                         }
                     }
-
                     stage('Run Experiments') {
                         steps {
                             script {
-- 
GitLab


From 2ce0078a56abd3d3fe8578d0408ab089e0e0a737 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 01:26:46 +0000
Subject: [PATCH 194/447] changed Cases to Case and fixed one that was a
 mismatch

---
 Jenkinsfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9aa494c35..d1911b06e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -59,7 +59,7 @@ pipeline {
                 agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
-                        name "Cases"
+                        name "Case"
                         values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
@@ -70,14 +70,16 @@ pipeline {
                                  env.HOME = "$HOME"
                                  env.RUNTESTS = "${HOME}/RUNTESTS"
                             }
-                            echo "Cases: ${Cases} ${HOME}"
-                            sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Cases}.yaml'
+                            echo "Case: ${Case} ${HOME}"
+                            sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml'
                         }
                     }
                     stage('Run Experiments') {
                         steps {
                             script {
+                                env.HOME = "$HOME"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                env.pslot = "$pslot"
                                 //pullRequest.removeLabel('CI-${machine}-Building')
                                 //pullRequest.addLabel('CI-${machine}-Running')
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-- 
GitLab


From aa20993a4b956a9b6e323518d9f16d20d1b7e26a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 01:29:42 +0000
Subject: [PATCH 195/447] replaced WORKSPACE holdover to HOME

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d1911b06e..379eafcf0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -84,7 +84,7 @@ pipeline {
                                 //pullRequest.addLabel('CI-${machine}-Running')
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             }
-                            sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
+                            sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                             script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") }
                         }
                     }
-- 
GitLab


From 29ee450803ed4db5989b9fe7215231ee64dd05ba Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 01:46:44 +0000
Subject: [PATCH 196/447] clean workspace

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 379eafcf0..eaba64f00 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,12 +40,13 @@ pipeline {
                     //pullRequest.addLabel("CI-${machine}-Building")
                 }
                 echo "Do Build for ${machine}"
+                cleanWs()
                 checkout scm
                 script {
                     HOME = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                 }
-                //sh 'sorc/build_all.sh -gu'
+                sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
                 sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
             }
-- 
GitLab


From 943a33e195a2ba98adc839e7b66f65b55a3f59db Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 17:21:51 +0000
Subject: [PATCH 197/447] Added PR Label setting for Running in non matrixed
 build block and added checks for Running on post

---
 Jenkinsfile | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index eaba64f00..f0099fb9d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -36,8 +36,8 @@ pipeline {
             steps {
                 script {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                    //pullRequest.removeLabel("CI-${machine}-Ready")
-                    //pullRequest.addLabel("CI-${machine}-Building")
+                    pullRequest.removeLabel("CI-${machine}-Ready")
+                    pullRequest.addLabel("CI-${machine}-Building")
                 }
                 echo "Do Build for ${machine}"
                 cleanWs()
@@ -49,6 +49,10 @@ pipeline {
                 sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
                 sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
+                script {
+                    pullRequest.removeLabel("CI-${machine}-Building")
+                    pullRequest.addLabel("CI-${machine}-Running")
+                }
             }
         }
 
@@ -81,8 +85,6 @@ pipeline {
                                 env.HOME = "$HOME"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 env.pslot = "$pslot"
-                                //pullRequest.removeLabel('CI-${machine}-Building')
-                                //pullRequest.addLabel('CI-${machine}-Running')
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             }
                             sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
@@ -97,16 +99,18 @@ pipeline {
     post {
         success {
             script {
-                echo "SUCCESS"
-                //pullRequest.removeLabel('CI-${machine}-Running')
-                //pullRequest.addLabel('CI-${machine}-Passed')  
+                if(pullRequest.labels.contains("CI-${machine}-Running")) {
+                   pullRequest.removeLabel('CI-${machine}-Running')
+                   pullRequest.addLabel('CI-${machine}-Passed')
+                }
             }
         }
         failure {
             script {
-                echo "FAILURE"
-                //pullRequest.removeLabel('CI-${machine}-Running')
-                //pullRequest.addLabel('CI-${machine}-Failed')  
+                if(pullRequest.labels.contains("CI-${machine}-Running")) {
+                   pullRequest.removeLabel('CI-${machine}-Running')
+                   pullRequest.addLabel('CI-${machine}-Failed')
+                }
             }
         }
     }
-- 
GitLab


From 95c24316ea706ba404c5555a0a833fd8a618f307 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 17:25:37 +0000
Subject: [PATCH 198/447] remove old referencd Jenkinsfiles

---
 Jenkins_arch_opt | 35 --------------------
 Jenkinsfile_main | 85 ------------------------------------------------
 2 files changed, 120 deletions(-)
 delete mode 100644 Jenkins_arch_opt
 delete mode 100644 Jenkinsfile_main

diff --git a/Jenkins_arch_opt b/Jenkins_arch_opt
deleted file mode 100644
index f10858de7..000000000
--- a/Jenkins_arch_opt
+++ /dev/null
@@ -1,35 +0,0 @@
-pipeline {
-    agent none
-
-    stages {
-        parallel {
-            // The matrix directive would only be available
-            // in parallel blocks and is used to generate stages in
-            // a declarative fashion
-            matrix {
-                // Options define a map of options we could potentially have,
-                // each map defines a matrix for each individual stage
-                opts: [
-                    [machine: 'emc-orion'],
-                    [machine: 'emc-hera'],
-                ]
-            }
-
-            // To access variables defined in our opts we can use the groovy
-            // standard `it`
-            stage("${it.machine} Building & Testing") {
-                agent { label "${it.machine}" }
-                stage("Build") {
-                    steps {
-                        sh("echo Build on ${it.machine}")
-                    }
-                }
-                stage("Run tests") {
-                    steps {
-                        sh("Running Test One  on ${it.machine}")
-                    }
-                }
-            }    
-        }
-    }
-}
\ No newline at end of file
diff --git a/Jenkinsfile_main b/Jenkinsfile_main
deleted file mode 100644
index 142c5bb0a..000000000
--- a/Jenkinsfile_main
+++ /dev/null
@@ -1,85 +0,0 @@
-
-pipeline {
-    agent{ label 'orion-emc'}
-
-    stages {
-
-        stage('Checkout') {
-        agent{ label 'orion-emc'}
-            steps {
-                checkout scm
-                script {
-                   pullRequest.removeLabel('CI-Orion-Ready')
-                   pullRequest.addLabel('CI-Orion-Building')
-                }
-                sh 'git submodule update --init --recursive'
-            }
-        }
-
-        stage('Build') {
-        agent{ label 'orion-emc'}
-          steps {
-            sh 'sorc/build_all.sh -gu'
-            sh 'sorc/link_workflow.sh'
-          }
-        }
- 
-        stage('Create Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                sh 'rm -Rf ${WORKSPACE}/RUNTESTS'
-                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
-                script {
-                    pullRequest.removeLabel('CI-Orion-Building')
-                    pullRequest.addLabel('CI-Orion-Running')
-                    case_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pr_case_list", returnStdout: true ).trim()
-                    cases = case_list.tokenize('\n')
-                    cases.each { case_name ->
-                        stage("Create ${case_name}") {
-                            agent{ label 'orion-emc'}
-                              script { 
-                                env.case = case_name
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
-                              }
-                              sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS creating cases: ${cases} on Orion") }
-                }
-            }
-        }
-
-        stage('Run Experiments') {
-        agent{ label 'orion-emc'}
-            steps {
-                script {
-                    experiment_list = sh( script: "${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot_list ${WORKSPACE}/RUNTESTS", returnStdout: true ).trim()
-                    experiments = experiment_list.tokenize('\n')
-                    experiments.each { experiment_name ->
-                        stage("Run ${experiment_name}") {
-                            agent{ label 'orion-emc'}
-                              script { env.experiment = experiment_name }
-                              sh '${WORKSPACE}/ci/scripts/run-check_ci.sh ${WORKSPACE} ${experiment}'
-                        }
-                    }
-                    script { pullRequest.comment("SUCCESS running experiments: ${experiments} on Orion") }
-                }
-            }
-        }
-    }    
-
-    post {
-        success {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Passed')  
-            }
-        }
-        failure {
-            script {
-                pullRequest.removeLabel('CI-Orion-Running')
-                pullRequest.addLabel('CI-Orion-Failed')  
-            }
-        }
-    }
-}
\ No newline at end of file
-- 
GitLab


From 75d1db4bff7b4a1ccb658f0c1180305f9f27d953 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Jan 2024 17:28:07 +0000
Subject: [PATCH 199/447] removed another Jenkins reference file

---
 Jenkinsfile_arch_opt | 99 --------------------------------------------
 1 file changed, 99 deletions(-)
 delete mode 100644 Jenkinsfile_arch_opt

diff --git a/Jenkinsfile_arch_opt b/Jenkinsfile_arch_opt
deleted file mode 100644
index c35947d67..000000000
--- a/Jenkinsfile_arch_opt
+++ /dev/null
@@ -1,99 +0,0 @@
-pipeline {
-    agent none
-
-    options {
-        disableConcurrentBuilds()
-        overrideIndexTriggers(false)
-        skipDefaultCheckout(true)
-    }
-
-    stages {
-
-        stage( 'Get Machine' ) {
-
-            steps {
-                script {
-                    MACHINE = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches("CI-Hera-Ready"))) {
-                             MACHINE='hera'
-                        }  
-                        else if ((label.matches("CI-Orion-Ready"))) {
-                            MACHINE='orion'
-                        }  
-                        else if ((label.matches("CI-Hercules-Ready"))) {
-                            MACHINE='hercules'
-                        }  
-                     }
-                }
-            }
-        }
-
-        stage( 'Build and Test' ) {
-
-            when {
-                expression { MACHINE != 'none' }
-            }
-
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                axes {
-                    axis {
-                        name 'Cases'
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW',  'C96_atm3DVar'
-                    }
-                }
-                stages {
-                    stage('Build') {
-                        steps {
-                           script {
-                            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                            pullRequest.removeLabel("CI-${machine}-Ready")
-                            pullRequest.addLabel("CI-${machine}-Building")
-                           }
-                           echo "Do Build for ${machine} ${Cases}"
-                            checkout scm
-                            sh 'sorc/build_all.sh -gu'
-                            script { env.MACHINE_ID = MACHINE } 
-                            sh 'sorc/link_workflow.sh'
-                        }
-                    }
-                    stage('Create Experiment') {
-                        steps {
-                            script { 
-                                env.case = "${Cases}"
-                                env.RUNTESTS = "${WORKSPACE}/RUNTESTS"
-                            }
-                            sh 'mkdir -p ${RUNTESTS}'
-                            sh '${WORKSPACE}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ci/cases/pr/${case}.yaml'
-                        }
-                    }
-                    stage("Run Cases") {
-                        steps {
-                            echo "Do Test for ${machine} - ${Cases}"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-    
-post {
-    success {
-        script {
-            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-            echo "Do Post for ${machine}"
-            pullRequest.removeLabel("CI-${machine}-Building")
-            pullRequest.addLabel("CI-${machine}-Passed")
-        }
-    }
-    failure {
-        script {
-            machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-            pullRequest.removeLabel('CI-${machine}-Running')
-            pullRequest.addLabel('CI-${machine}-Failed')  
-        }
-    }
-}
\ No newline at end of file
-- 
GitLab


From 8a34f421b5034642061dfda3560539b38d4de2a7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Jan 2024 17:15:04 +0000
Subject: [PATCH 200/447] updated cancel_slurm to a generlized cancel_batch for
 pbs on wcoss

---
 ci/scripts/utils/ci_utils.sh | 46 ++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 5716654c1..e125fb54b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -1,28 +1,44 @@
 #!/bin/env bash
 
-function cancel_slurm_jobs() {
-
-  # Usage: cancel_slurm_jobs <substring>
-  # Example: cancel_slurm_jobs "C48_ATM_3c4e7f74"
+function cancel_batch_jobs() {
+  # Usage: cancel_batch_jobs <substring>
+  # Example: cancel_batch_jobs "C48_ATM_3c4e7f74"
   #
-  # Cancel all Slurm jobs that have the given substring in their name
+  # Cancel all batch jobs that have the given substring in their name
   # So like in the example all jobs with "C48_ATM_3c4e7f74"
   # in their name will be canceled
 
   local substring=$1
   local job_ids
-  job_ids=$(squeue -u "${USER}" -h -o "%i")
-
-  for job_id in ${job_ids}; do
-    job_name=$(sacct -j "${job_id}" --format=JobName%100 | head -3 | tail -1 | sed -r 's/\s+//g') || true
-    if [[ "${job_name}" =~ ${substring} ]]; then
-      echo "Canceling Slurm Job ${job_name} with: scancel ${job_id}"
-      scancel "${job_id}"
-      continue
-    fi
-  done
+
+  # cancel pbs jobs <substring>
+  if [[ ${MACHINE_ID} == "wcoss2" ]]; then
+    job_ids=$(qstat -u "${USER}" | awk '{print $1}')
+
+    for job_id in ${job_ids}; do
+      job_name=$(qstat -f "${job_id}" | grep Job_Name | awk '{print $3}') || true
+      if [[ "${job_name}" =~ ${substring} ]]; then
+        echo "Canceling PBS Job ${job_name} with: qdel ${job_id}"
+        qdel "${job_id}"
+        continue
+      fi
+    done
+  # cancel slurm jobs <substring>
+  else 
+    job_ids=$(squeue -u "${USER}" -h -o "%i")
+
+    for job_id in ${job_ids}; do
+      job_name=$(sacct -j "${job_id}" --format=JobName%100 | head -3 | tail -1 | sed -r 's/\s+//g') || true
+      if [[ "${job_name}" =~ ${substring} ]]; then
+        echo "Canceling Slurm Job ${job_name} with: scancel ${job_id}"
+        scancel "${job_id}"
+        continue
+      fi
+    done
+  fi
 }
 
+
 function get_pr_case_list () {
 
     #############################################################
-- 
GitLab


From 1d74f242f591226de8e98bc86ab97dec0e3f8f39 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 20:32:51 +0000
Subject: [PATCH 201/447] added some post clean up and messaging

---
 Jenkinsfile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f0099fb9d..d24088835 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,7 +3,7 @@ pipeline {
     agent none
     options {
         disableConcurrentBuilds(abortPrevious: true)
-        skipDefaultCheckout()
+        skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
@@ -103,7 +103,10 @@ pipeline {
                    pullRequest.removeLabel('CI-${machine}-Running')
                    pullRequest.addLabel('CI-${machine}-Passed')
                 }
+                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
             }
+            cleanWs()
         }
         failure {
             script {
@@ -111,6 +114,8 @@ pipeline {
                    pullRequest.removeLabel('CI-${machine}-Running')
                    pullRequest.addLabel('CI-${machine}-Failed')
                 }
+                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
             }
         }
     }
-- 
GitLab


From 143c196db5e2904d37be19560661fa6f3ff55514 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 22:16:38 +0000
Subject: [PATCH 202/447] needed Case to be in shell env as well as in Jenkins
 script environment (turned off clean and build in build block

---
 Jenkinsfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d24088835..ae83fd13c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,6 @@
 pipeline {
 
-    agent none
+    agent { label 'built-in' }
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
@@ -40,13 +40,13 @@ pipeline {
                     pullRequest.addLabel("CI-${machine}-Building")
                 }
                 echo "Do Build for ${machine}"
-                cleanWs()
+                // cleanWs()
                 checkout scm
                 script {
                     HOME = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                 }
-                sh 'sorc/build_all.sh -gu'
+                // sh 'sorc/build_all.sh -gu'
                 sh 'sorc/link_workflow.sh'
                 sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
                 script {
@@ -83,6 +83,7 @@ pipeline {
                         steps {
                             script {
                                 env.HOME = "$HOME"
+                                env.Case = "$Case"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 env.pslot = "$pslot"
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-- 
GitLab


From d7d6a79bb8b1958a7ed729d82d3bf1bfec42124f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 22:33:02 +0000
Subject: [PATCH 203/447] still not getting the correct pslot in the matrix,
 remove shell reference and assigned shell env to variable directly

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ae83fd13c..04728b519 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,7 +72,7 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                             script {
-                                 env.HOME = "$HOME"
+                                 env.HOME = HOME
                                  env.RUNTESTS = "${HOME}/RUNTESTS"
                             }
                             echo "Case: ${Case} ${HOME}"
@@ -82,10 +82,10 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                             script {
-                                env.HOME = "$HOME"
-                                env.Case = "$Case"
+                                env.HOME = "${HOME}"
+                                env.Case = "${Case}"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                env.pslot = "$pslot"
+                                env.pslot = pslot
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             }
                             sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
-- 
GitLab


From fcf18d13125ab2f6e5a633eba048d081395fad14 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 22:48:29 +0000
Subject: [PATCH 204/447] still not getting the correct pslot in the matrix,
 moved shell sh into Jenkins script block

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 04728b519..56ed743c2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -87,9 +87,10 @@ pipeline {
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 env.pslot = pslot
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                                sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             }
-                            sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
-                            script { pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}") }
+                            // sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                         }
                     }
                 }
-- 
GitLab


From cc3905210cd1d9ac6cfde4b8c1f485a16b64a2ca Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 22:52:48 +0000
Subject: [PATCH 205/447] finally getting pslot in the shell, checking if env
 statements are needed

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 56ed743c2..1f96737a9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -82,10 +82,10 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                             script {
-                                env.HOME = "${HOME}"
-                                env.Case = "${Case}"
+                                //env.HOME = "${HOME}"
+                                //env.Case = "${Case}"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                env.pslot = pslot
+                                //env.pslot = pslot
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                 sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
-- 
GitLab


From fc4dcb2fe0aee4767608f23d7ec0e490948a5a0e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 23:00:15 +0000
Subject: [PATCH 206/447] remove env assignment statements since all shell
 commands to the workspace is now done with Jenkins script commands

---
 Jenkinsfile | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1f96737a9..dc89b5f17 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -82,15 +82,11 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                             script {
-                                //env.HOME = "${HOME}"
-                                //env.Case = "${Case}"
                                 pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                //env.pslot = pslot
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                 sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             }
-                            // sh '${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}'
                         }
                     }
                 }
-- 
GitLab


From 0d535d214e97c6fd0b40fb75dbcb1a430ce1064b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 23:29:35 +0000
Subject: [PATCH 207/447] consolodated all the bash shells into Jenkins scripts
 with the sh method to force it to be declartive

---
 Jenkinsfile | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dc89b5f17..06131389f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,18 +38,13 @@ pipeline {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
-                }
-                echo "Do Build for ${machine}"
-                // cleanWs()
-                checkout scm
-                script {
-                    HOME = "${WORKSPACE}"
+                    cleanWs()
+                    checkout scm
+                    HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
-                }
-                // sh 'sorc/build_all.sh -gu'
-                sh 'sorc/link_workflow.sh'
-                sh 'mkdir -p ${WORKSPACE}/RUNTESTS'
-                script {
+                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    sh( script: "sorc/link_workflow.sh", returnStatus: false)
+                    sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
@@ -69,29 +64,31 @@ pipeline {
                     }
                 }
                 stages {
+                  ws("${HOMEgfs}") {
                     stage('Create Experiment') {
                         steps {
                             script {
-                                 env.HOME = HOME
-                                 env.RUNTESTS = "${HOME}/RUNTESTS"
+                                env.HOME = HOMEgfs
+                                env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
+                                sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                             }
-                            echo "Case: ${Case} ${HOME}"
-                            sh '${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml'
                         }
                     }
                     stage('Run Experiments') {
                         steps {
                             script {
-                                pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             }
                         }
                     }
+                  }
                 }
             }
-        }
+       }
+
     }
 
     post {
-- 
GitLab


From 07ffbf42111da092a8509531dced2759c4f96570 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Jan 2024 23:48:29 +0000
Subject: [PATCH 208/447] moved ws under steps

---
 Jenkinsfile | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 06131389f..8dd9ecf46 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -64,27 +64,28 @@ pipeline {
                     }
                 }
                 stages {
-                  ws("${HOMEgfs}") {
                     stage('Create Experiment') {
                         steps {
-                            script {
-                                env.HOME = HOMEgfs
-                                env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
-                                sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                            ws("${HOMEgfs}") {
+                                script {
+                                    env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
+                                    sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                                }
                             }
                         }
                     }
                     stage('Run Experiments') {
                         steps {
-                            script {
-                                pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                            ws("${HOMEgfs}") {
+                                script {
+                                    pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                    pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                                    sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                               }
                             }
                         }
                     }
-                  }
                 }
             }
        }
-- 
GitLab


From d70b7acf249e743693f4106904034be07162599c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 00:50:40 +0000
Subject: [PATCH 209/447] moved HOME and ws around

---
 Jenkinsfile | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8dd9ecf46..616f7f91b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
-                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
@@ -66,21 +66,23 @@ pipeline {
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            ws("${HOMEgfs}") {
+                            ws(HOMEgfs) {
                                 script {
-                                    env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
-                                    sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                                    env.HOME=HOMEgfs
+                                    env.RUNTESTS = "${HOME}/RUNTESTS"
+                                    sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
                             }
                         }
                     }
                     stage('Run Experiments') {
                         steps {
-                            ws("${HOMEgfs}") {
+                            ws(HOMEgfs) {
                                 script {
-                                    pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                    env.HOME=HOMEgfs
+                                    pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    sh( script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
-- 
GitLab


From e15b033276286669b80428b4d92822061fe89d42 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:01:48 +0000
Subject: [PATCH 210/447] added HOME for common build into full paths of
 scripts

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 616f7f91b..2c4ca55c3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -70,7 +70,7 @@ pipeline {
                                 script {
                                     env.HOME=HOMEgfs
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                                    sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
                             }
                         }
@@ -80,9 +80,9 @@ pipeline {
                             ws(HOMEgfs) {
                                 script {
                                     env.HOME=HOMEgfs
-                                    pslot = sh( script: "ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                    pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                    sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
-- 
GitLab


From 2d0c5008515e65cb686813196470db9dc85ab33e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:12:28 +0000
Subject: [PATCH 211/447] added global envirment trying to track down where
 decltartive fails

---
 Jenkinsfile | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2c4ca55c3..dbce61554 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,6 +7,11 @@ pipeline {
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
+    environment {
+        HOMEgfs = "${WORKSPACE}"
+        MACHINE = 'none'
+    }
+
     stages {
 
         stage('Get Machine') {
@@ -68,7 +73,6 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    env.HOME=HOMEgfs
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
@@ -79,7 +83,6 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    env.HOME=HOMEgfs
                                     pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
-- 
GitLab


From 7e17063a8867ab91835564fae90038bea22e4510 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:24:56 +0000
Subject: [PATCH 212/447] removed glogal environment and when blocks

---
 Jenkinsfile | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dbce61554..7efac8247 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,11 +7,6 @@ pipeline {
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
-    environment {
-        HOMEgfs = "${WORKSPACE}"
-        MACHINE = 'none'
-    }
-
     stages {
 
         stage('Get Machine') {
@@ -35,9 +30,9 @@ pipeline {
 
         stage('Build') {
             agent { label "${MACHINE}-emc" }
-            when {
-                expression { MACHINE != 'none' }
-            }
+            //when {
+            //    expression { MACHINE != 'none' }
+            //}
             steps {
                 script {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
@@ -57,9 +52,9 @@ pipeline {
         }
 
         stage('Run Tests') {
-            when {
-                expression { MACHINE != 'none' }
-            }
+            //when {
+            //    expression { MACHINE != 'none' }
+            //}
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
-- 
GitLab


From f30c678e89f052ae25b97d312d4ce384e4d38258 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:32:13 +0000
Subject: [PATCH 213/447] added HOMEgfs in shell (not sh) because it was
 picking up /home/tmcguinness from sh

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7efac8247..bd4e401e9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,8 +68,8 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOME}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                                    env.RUNTESTS = "${gfsHOME}/RUNTESTS"
+                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
                             }
                         }
@@ -78,9 +78,9 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    pslot = sh( script: "${HOME}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "${HOME}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
-- 
GitLab


From b62fac741d19876ce615a8080442e2a4e0c3474c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:40:51 +0000
Subject: [PATCH 214/447] oops had a misspelling of gfsHOME

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bd4e401e9..70fcb5368 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,7 +68,7 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    env.RUNTESTS = "${gfsHOME}/RUNTESTS"
+                                    env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
                             }
-- 
GitLab


From 129c13537175f3138eb7dc03678189ab11c959ab Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 01:53:37 +0000
Subject: [PATCH 215/447] put the actual build back in

---
 Jenkinsfile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 70fcb5368..f26734011 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -30,9 +30,9 @@ pipeline {
 
         stage('Build') {
             agent { label "${MACHINE}-emc" }
-            //when {
-            //    expression { MACHINE != 'none' }
-            //}
+            when {
+                expression { MACHINE != 'none' }
+            }
             steps {
                 script {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
@@ -42,7 +42,7 @@ pipeline {
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
-                    //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
@@ -52,9 +52,9 @@ pipeline {
         }
 
         stage('Run Tests') {
-            //when {
-            //    expression { MACHINE != 'none' }
-            //}
+            when {
+                expression { MACHINE != 'none' }
+            }
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
-- 
GitLab


From 77d20dd758bff9e41198aba55808023093a12e51 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 17 Jan 2024 10:48:57 -0500
Subject: [PATCH 216/447] commented out sm and build

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f26734011..459267dfe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,11 +38,11 @@ pipeline {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
-                    cleanWs()
-                    checkout scm
+                    //cleanWs()
+                    //checkout scm
                     HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
-                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
-- 
GitLab


From c9c37229581c1a856e1b06c25f04a68807039bd6 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 17 Jan 2024 10:57:31 -0500
Subject: [PATCH 217/447] put back in scm and checkout to prime script
 locations on restart

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 459267dfe..c217dc27c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,8 +38,8 @@ pipeline {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
-                    //cleanWs()
-                    //checkout scm
+                    cleanWs()
+                    checkout scm
                     HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
                     //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-- 
GitLab


From 64a37db13edf45455d46b5a234453126ea400578 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Jan 2024 16:51:02 +0000
Subject: [PATCH 218/447] made two shared values between stages global in
 attempts to make pipeline declartive

---
 Jenkinsfile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c217dc27c..daba1188a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,3 +1,6 @@
+def MACHINE = 'none'
+def HOMEgfs = 'initial value'
+
 pipeline {
 
     agent { label 'built-in' }
@@ -42,7 +45,7 @@ pipeline {
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
                     env.MACHINE_ID = MACHINE
-                    //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
@@ -60,7 +63,8 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values "C48_ATM", "C48_S2SW"
                     }
                 }
                 stages {
-- 
GitLab


From a7ac197f24c330bb4a81b65beae4a80e74af0e4d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Thu, 18 Jan 2024 20:38:56 +0000
Subject: [PATCH 219/447] Update ci_utils.sh

or true for qstat for job_ids shell check
---
 ci/scripts/utils/ci_utils.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index e125fb54b..a0b45d5d5 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -13,7 +13,7 @@ function cancel_batch_jobs() {
 
   # cancel pbs jobs <substring>
   if [[ ${MACHINE_ID} == "wcoss2" ]]; then
-    job_ids=$(qstat -u "${USER}" | awk '{print $1}')
+    job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do
       job_name=$(qstat -f "${job_id}" | grep Job_Name | awk '{print $3}') || true
@@ -99,4 +99,4 @@ function create_experiment () {
   export pslot=${case}_${pr_sha}
   "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
-}
\ No newline at end of file
+}
-- 
GitLab


From 154dd055f1c1ad9e502004aef34c0d2487619966 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 21:52:25 +0000
Subject: [PATCH 220/447] working on state added BUILD semaphore file in sorc
 dir

---
 Jenkinsfile                     | 35 ++++++++++++++++++++++-----------
 ci/scripts/run-check_ci_stub.sh |  5 +++++
 sorc/build_all_stub.sh          |  5 +++++
 3 files changed, 33 insertions(+), 12 deletions(-)
 create mode 100644 ci/scripts/run-check_ci_stub.sh
 create mode 100644 sorc/build_all_stub.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index daba1188a..88f5bba45 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -33,9 +33,9 @@ pipeline {
 
         stage('Build') {
             agent { label "${MACHINE}-emc" }
-            when {
-                expression { MACHINE != 'none' }
-            }
+            //when {
+            //    expression { MACHINE != 'none' }
+            //}
             steps {
                 script {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
@@ -44,8 +44,18 @@ pipeline {
                     cleanWs()
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
+                }
+                script {
                     env.MACHINE_ID = MACHINE
-                    sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                    if (not fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
+                        //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                        sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
+                    }
+                    else {
+                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
+                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                    }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
@@ -55,9 +65,9 @@ pipeline {
         }
 
         stage('Run Tests') {
-            when {
-                expression { MACHINE != 'none' }
-            }
+            //when {
+            //    expression { MACHINE != 'none' }
+            //}
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
@@ -84,7 +94,8 @@ pipeline {
                                 script {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: false)
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
@@ -100,9 +111,9 @@ pipeline {
         success {
             script {
                 if(pullRequest.labels.contains("CI-${machine}-Running")) {
-                   pullRequest.removeLabel('CI-${machine}-Running')
-                   pullRequest.addLabel('CI-${machine}-Passed')
+                   pullRequest.removeLabel("CI-${machine}-Running")
                 }
+                pullRequest.addLabel("CI-${machine}-Passed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
             }
@@ -111,9 +122,9 @@ pipeline {
         failure {
             script {
                 if(pullRequest.labels.contains("CI-${machine}-Running")) {
-                   pullRequest.removeLabel('CI-${machine}-Running')
-                   pullRequest.addLabel('CI-${machine}-Failed')
+                   pullRequest.removeLabel("CI-${machine}-Running")
                 }
+                pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
             }
diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
new file mode 100644
index 000000000..f2903b9a2
--- /dev/null
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eu
+
+echo -e "HOMEgfs: $1\npslpot: $2\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
\ No newline at end of file
diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh
new file mode 100644
index 000000000..9acf802c4
--- /dev/null
+++ b/sorc/build_all_stub.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eu
+
+echo "Ran build all stub in ${PWD}" > BUILD_STUB_RAN_$(date +%d%H%M)
\ No newline at end of file
-- 
GitLab


From 82627415736d8d3ad493338eb398c2a103bed9f3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 21:58:58 +0000
Subject: [PATCH 221/447] reversed logic for BUILT file semaphore

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 88f5bba45..22495ac7b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,15 +47,15 @@ pipeline {
                 }
                 script {
                     env.MACHINE_ID = MACHINE
-                    if (not fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
+                    if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
+                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
+                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                    }
+                    else {
                         //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                         sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
                     }
-                    else {
-                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
-                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                    }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
-- 
GitLab


From 5a0c52abbb9560e5cdc492c9c856d922e9ae06fe Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 22:05:47 +0000
Subject: [PATCH 222/447] added chmod x to sub scripts

---
 ci/scripts/run-check_ci_stub.sh | 0
 sorc/build_all_stub.sh          | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 ci/scripts/run-check_ci_stub.sh
 mode change 100644 => 100755 sorc/build_all_stub.sh

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
old mode 100644
new mode 100755
diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh
old mode 100644
new mode 100755
-- 
GitLab


From bf8896bf43da1b02fa57ef8764867ff47208beb4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 22:36:56 +0000
Subject: [PATCH 223/447] needed to set HOMEgfs in run stub

---
 ci/scripts/run-check_ci_stub.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index f2903b9a2..191bce5d9 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -2,4 +2,7 @@
 
 set -eu
 
-echo -e "HOMEgfs: $1\npslpot: $2\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
\ No newline at end of file
+HOMEgfs=$1
+pslpot=$2
+
+echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
\ No newline at end of file
-- 
GitLab


From 7e60d2357f0d58e4144907e65f182752a42780c8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 22:54:50 +0000
Subject: [PATCH 224/447] needed to set HOMEgfs in run stub and looking at
 pullRequest.labels.contains

---
 Jenkinsfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 22495ac7b..0d67414af 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,6 +50,9 @@ pipeline {
                     if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
                         HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                        //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                        sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
                     }
                     else {
                         //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
@@ -110,7 +113,7 @@ pipeline {
     post {
         success {
             script {
-                if(pullRequest.labels.contains("CI-${machine}-Running")) {
+                if ( pullRequest.labels.contains( 'CI-${machine}-Running') ) {
                    pullRequest.removeLabel("CI-${machine}-Running")
                 }
                 pullRequest.addLabel("CI-${machine}-Passed")
-- 
GitLab


From 12e3126e5908898c5c0027ac193c89a980cc947e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 23:01:14 +0000
Subject: [PATCH 225/447] typoed pslot in run check stub script

---
 ci/scripts/run-check_ci_stub.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index 191bce5d9..01975f566 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -3,6 +3,6 @@
 set -eu
 
 HOMEgfs=$1
-pslpot=$2
+pslot=$2
 
 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
\ No newline at end of file
-- 
GitLab


From 8f8e8922fa15e415ef6b1bb0cdfd43c5a64a0dda Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 23:40:19 +0000
Subject: [PATCH 226/447] hard coded node name

---
 Jenkinsfile | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0d67414af..9b2c5e4b6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,7 +32,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "${MACHINE}-emc" }
+            agent { label "Orion-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -41,7 +41,6 @@ pipeline {
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
-                    cleanWs()
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
                 }
@@ -50,9 +49,6 @@ pipeline {
                     if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
                         HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                        //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                        sh( script: "sorc/build_all_stub.sh", returnStatus: false)
-                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
                     }
                     else {
                         //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
@@ -72,7 +68,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "${MACHINE}-emc" }
+                agent { label "Orion-emc" }
                 axes {
                     axis {
                         name "Case"
@@ -113,9 +109,10 @@ pipeline {
     post {
         success {
             script {
-                if ( pullRequest.labels.contains( 'CI-${machine}-Running') ) {
+                if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) {
                    pullRequest.removeLabel("CI-${machine}-Running")
                 }
+                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Passed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
@@ -126,7 +123,8 @@ pipeline {
             script {
                 if(pullRequest.labels.contains("CI-${machine}-Running")) {
                    pullRequest.removeLabel("CI-${machine}-Running")
-                }
+                } 
+                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-- 
GitLab


From 94f841767d58fc5cfde30004bbe18e802f2d6da0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Jan 2024 23:53:26 +0000
Subject: [PATCH 227/447] hard coded node names typo capped

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9b2c5e4b6..d9ad451bd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,7 +32,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "Orion-emc" }
+            agent { label "orion-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -68,7 +68,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "Orion-emc" }
+                agent { label "orion-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From 0489383ac058840edb1fecd693221e95e657e240 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 00:13:47 +0000
Subject: [PATCH 228/447] returnStdout on cat BUIT semaphore didnt save

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d9ad451bd..d67432013 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,12 +47,12 @@ pipeline {
                 script {
                     env.MACHINE_ID = MACHINE
                     if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
-                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true).trim()
+                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
                         //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                        sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                        sh( script: "sorc/build_all_stub.sh" )
                         sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
-- 
GitLab


From 68d245b02c0b7bbe6f4a6ae02bfa5b7c844a1e94 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 16:55:47 +0000
Subject: [PATCH 229/447] added node properties

---
 Jenkinsfile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index d67432013..7b580b1fd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,6 +3,12 @@ def HOMEgfs = 'initial value'
 
 pipeline {
 
+    properties([parameters([[$class: 'NodeParameterDefinition',
+       allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+       defaultSlaves: ['built-in'], name: 'rdhpcs',
+       nodeEligibility: [$class: 'AllNodeEligibility'],
+       triggerIfResult: 'allCases']])])
+
     agent { label 'built-in' }
     options {
         disableConcurrentBuilds(abortPrevious: true)
-- 
GitLab


From 0834c3f93c803e4b8b37ed7f919c809efef47c18 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:14:50 +0000
Subject: [PATCH 230/447] restored configurable node

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7b580b1fd..9d7cbf59d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,7 +38,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "orion-emc" }
+            agent { label "${machine}-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -74,7 +74,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "orion-emc" }
+                agent { label "${machine}-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From d76e1f1d8cff334de3d0ffbc5fdd007e3e62ddfb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:26:19 +0000
Subject: [PATCH 231/447] moved parameters into options block

---
 Jenkinsfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9d7cbf59d..b42920cb9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,14 +3,13 @@ def HOMEgfs = 'initial value'
 
 pipeline {
 
-    properties([parameters([[$class: 'NodeParameterDefinition',
-       allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-       defaultSlaves: ['built-in'], name: 'rdhpcs',
-       nodeEligibility: [$class: 'AllNodeEligibility'],
-       triggerIfResult: 'allCases']])])
-
     agent { label 'built-in' }
     options {
+       parameters([[$class: 'NodeParameterDefinition',
+            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+            defaultSlaves: ['built-in'], name: 'rdhpcs',
+            nodeEligibility: [$class: 'AllNodeEligibility'],
+            triggerIfResult: 'allCases']])
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
-- 
GitLab


From 5471c46eaef99e12ac6958077af89db5fdd30776 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:36:40 +0000
Subject: [PATCH 232/447] added just parameters section

---
 Jenkinsfile | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b42920cb9..db32cef5d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,14 +2,15 @@ def MACHINE = 'none'
 def HOMEgfs = 'initial value'
 
 pipeline {
-
     agent { label 'built-in' }
+
+    parameters{[[$class: 'NodeParameterDefinition',
+       allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+       name: 'rdhpcs',
+       nodeEligibility: [$class: 'AllNodeEligibility'],
+       triggerIfResult: 'allCases']]}
+
     options {
-       parameters([[$class: 'NodeParameterDefinition',
-            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-            defaultSlaves: ['built-in'], name: 'rdhpcs',
-            nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases']])
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
-- 
GitLab


From 8d7d4b2c173e0bb6d9e3ed78915396a0963d9548 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:40:34 +0000
Subject: [PATCH 233/447] moved parameters into options

---
 Jenkinsfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index db32cef5d..20f484559 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,13 +4,12 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-    parameters{[[$class: 'NodeParameterDefinition',
-       allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-       name: 'rdhpcs',
-       nodeEligibility: [$class: 'AllNodeEligibility'],
-       triggerIfResult: 'allCases']]}
-
     options {
+        parameters(class: 'NodeParameterDefinition',
+            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+            defaultSlaves: ['built-in'], name: 'rdhpcs',
+            nodeEligibility: [$class: 'AllNodeEligibility'],
+            triggerIfResult: 'allCases')
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
-- 
GitLab


From abb747009a7f9da567e0683566445d60794f75f3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:43:55 +0000
Subject: [PATCH 234/447] parameters can not be in options

---
 Jenkinsfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 20f484559..f1a544461 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -5,15 +5,16 @@ pipeline {
     agent { label 'built-in' }
 
     options {
-        parameters(class: 'NodeParameterDefinition',
-            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-            defaultSlaves: ['built-in'], name: 'rdhpcs',
-            nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases')
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
+    parameters{class: 'NodeParameterDefinition',
+            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+            defaultSlaves: ['built-in'], name: 'rdhpcs',
+            nodeEligibility: [$class: 'AllNodeEligibility'],
+            triggerIfResult: 'allCases'
+            }
 
     stages {
 
-- 
GitLab


From c46fdd68a7d3f79b0d57bbea8e0d6d7765108f13 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:48:37 +0000
Subject: [PATCH 235/447] removed class in parameters

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f1a544461..4ac5aba76 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -9,7 +9,7 @@ pipeline {
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
-    parameters{class: 'NodeParameterDefinition',
+    parameters{
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-- 
GitLab


From 52bd9736a94a13c64d925817b7664bcfe6dd9e59 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:51:45 +0000
Subject: [PATCH 236/447] added class in bracketrs

---
 Jenkinsfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4ac5aba76..ce16ef46e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,17 +4,16 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
+        parameters{[[$class: 'NodeParameterDefinition',
+            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
+            defaultSlaves: ['built-in'], name: 'rdhpcs',
+            nodeEligibility: [$class: 'AllNodeEligibility'],
+            triggerIfResult: 'allCases']]}
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
-    parameters{
-            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-            defaultSlaves: ['built-in'], name: 'rdhpcs',
-            nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases'
-            }
 
     stages {
 
-- 
GitLab


From 6d0eb3acb0163cadc953a369c1f61e8244c10eae Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:52:59 +0000
Subject: [PATCH 237/447] removed double bracket

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ce16ef46e..7d6fda8bc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,11 +4,11 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-        parameters{[[$class: 'NodeParameterDefinition',
+        parameters{[$class: 'NodeParameterDefinition',
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases']]}
+            triggerIfResult: 'allCases']}
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
-- 
GitLab


From b6f721241abd5f2b29092425dbd165a51cb71552 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 17:54:04 +0000
Subject: [PATCH 238/447] not sure how to hadd class

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7d6fda8bc..e6e92cb0f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,11 +4,11 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-        parameters{[$class: 'NodeParameterDefinition',
+        parameters{$class: 'NodeParameterDefinition',
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases']}
+            triggerIfResult: 'allCases'}
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
-- 
GitLab


From a94f2ce3575e03ffea846304c13b662d18021e19 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:02:19 +0000
Subject: [PATCH 239/447] still can not get syntax for parameters

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e6e92cb0f..4586a7cca 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,11 +4,12 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-        parameters{$class: 'NodeParameterDefinition',
+    parameters: [[ $class: 'NodeParameterDefinition',
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases'}
+            triggerIfResult: 'allCases' ]]
+
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
-- 
GitLab


From 9ccf7bc0e707ba3bd4eaa1eea421dd38c81fd8b0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:04:41 +0000
Subject: [PATCH 240/447] still not finding class

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4586a7cca..fe562d8cd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,11 +4,11 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-    parameters: [[ $class: 'NodeParameterDefinition',
+    parameters{ $class: 'NodeParameterDefinition',
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases' ]]
+            triggerIfResult: 'allCases'}
 
     options {
         disableConcurrentBuilds(abortPrevious: true)
-- 
GitLab


From f4edccea553156bf0252ab241987d52803170a15 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:08:41 +0000
Subject: [PATCH 241/447] just removed class line

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe562d8cd..c600591ad 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,7 +4,7 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-    parameters{ $class: 'NodeParameterDefinition',
+    parameters{ 
             allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
             defaultSlaves: ['built-in'], name: 'rdhpcs',
             nodeEligibility: [$class: 'AllNodeEligibility'],
-- 
GitLab


From bd4d49c00caca2049a8bf2eda0aba5b3c54a3cf5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:21:31 +0000
Subject: [PATCH 242/447] moved properties into stage

---
 Jenkinsfile | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c600591ad..c25ea9ee5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,12 +4,6 @@ def HOMEgfs = 'initial value'
 pipeline {
     agent { label 'built-in' }
 
-    parameters{ 
-            allowedSlaves: ['Hera-EMC', 'Orion-EMC'],
-            defaultSlaves: ['built-in'], name: 'rdhpcs',
-            nodeEligibility: [$class: 'AllNodeEligibility'],
-            triggerIfResult: 'allCases'}
-
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
@@ -73,6 +67,7 @@ pipeline {
             //when {
             //    expression { MACHINE != 'none' }
             //}
+            properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
             matrix {
                 agent { label "${machine}-emc" }
                 axes {
-- 
GitLab


From 45a8a4d014abcdc26228dfe295dab86ba0e7be99 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:23:52 +0000
Subject: [PATCH 243/447] moved properties into script in Run Seection

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c25ea9ee5..60f3b9a1c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -67,7 +67,6 @@ pipeline {
             //when {
             //    expression { MACHINE != 'none' }
             //}
-            properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
             matrix {
                 agent { label "${machine}-emc" }
                 axes {
@@ -92,6 +91,7 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
+            properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 5d29ceb49b8dd6984b148de7621bd7f7ea8f1c31 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:37:16 +0000
Subject: [PATCH 244/447] had machine instead of MACIHNE in node name

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 60f3b9a1c..8ac43a2bb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,7 +32,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "${machine}-emc" }
+            agent { label "${MACHINE}-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -68,7 +68,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "${machine}-emc" }
+                agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From 734d4fc663dce6a99de0967f2bca9ed48d906eb0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 18:57:18 +0000
Subject: [PATCH 245/447] added properties for epic nodes

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ac43a2bb..94c5c9ea8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -91,7 +91,7 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-            properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: [], defaultSlaves: [], name: 'rdhpc', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 39529e5f43927c7f9ce83dc5d01092f4ab5a7763 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 20:08:06 +0000
Subject: [PATCH 246/447] needed to remove RUNTESTS dir on rebuild

---
 Jenkinsfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 94c5c9ea8..af08e1d88 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -27,6 +27,7 @@ pipeline {
                             MACHINE = 'hercules'
                         }
                     }
+                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                 }
             }
         }
@@ -38,7 +39,6 @@ pipeline {
             //}
             steps {
                 script {
-                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
@@ -53,10 +53,11 @@ pipeline {
                     else {
                         //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                         sh( script: "sorc/build_all_stub.sh" )
-                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: false)
+                        sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true)
+                        sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true)
+                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                    sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: false)
                     pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
-- 
GitLab


From 9d5ab0448860bbc518601a22a71050488a78e6e6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 20:17:40 +0000
Subject: [PATCH 247/447] added exit 1 to run stub

---
 Jenkinsfile                     | 2 +-
 ci/scripts/run-check_ci_stub.sh | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index af08e1d88..713d7577f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -96,7 +96,7 @@ pipeline {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: true)
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index 01975f566..d98b87db2 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -5,4 +5,5 @@ set -eu
 HOMEgfs=$1
 pslot=$2
 
-echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
\ No newline at end of file
+echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
+exit 1
\ No newline at end of file
-- 
GitLab


From 4da660f6cea717d311cf53a0f358d94b84f93d30 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 20:29:23 +0000
Subject: [PATCH 248/447] removed returnstatus flag on run stub

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 713d7577f..17cde6781 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -96,7 +96,7 @@ pipeline {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}", returnStatus: true)
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
-- 
GitLab


From 92061568edf303796ef60c7ce47b650b3869cd12 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 20:48:24 +0000
Subject: [PATCH 249/447] Updated properties to my controler

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 17cde6781..eb77f601b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -92,7 +92,8 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                    //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 8715f5701c7fddf6c73bdd7a644efc7d411e4be7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:00:20 +0000
Subject: [PATCH 250/447] Can not get parameteres to sork again

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index eb77f601b..16c1260e7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -92,8 +92,8 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                    //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+                                    //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 859d7cb568a4307727b8615e87320a33695d4aca Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:05:32 +0000
Subject: [PATCH 251/447] added defaut slave as running one

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 16c1260e7..5d4234f61 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -93,7 +93,7 @@ pipeline {
                             ws(HOMEgfs) {
                                 script {
                                     //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 450bcab37afe615bef9b21957068a2ad96816498 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:07:57 +0000
Subject: [PATCH 252/447] added prop in create

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5d4234f61..aa71cedd4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -82,6 +82,7 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
-- 
GitLab


From fbbe569b14963a40beb1e32fe43a0bbac6d034d2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:16:55 +0000
Subject: [PATCH 253/447] added prop in create still brokek

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index aa71cedd4..8951bdd9c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -94,7 +94,7 @@ pipeline {
                             ws(HOMEgfs) {
                                 script {
                                     //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From a2226e2548b609fb77fc7e787069a80776ac6b9a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:21:07 +0000
Subject: [PATCH 254/447] still broken

---
 Jenkinsfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8951bdd9c..b9e18372f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -15,6 +15,8 @@ pipeline {
         stage('Get Machine') {
             agent { label 'built-in' }
             steps {
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+
                 script {
                     MACHINE = 'none'
                     for (label in pullRequest.labels) {
@@ -82,7 +84,6 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
@@ -93,8 +94,6 @@ pipeline {
                         steps {
                             ws(HOMEgfs) {
                                 script {
-                                    //properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 48a34d022915b5bc3c0a2f92d70075372561c52e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:23:34 +0000
Subject: [PATCH 255/447] moved prop under get machine script

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b9e18372f..d044503db 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -15,9 +15,8 @@ pipeline {
         stage('Get Machine') {
             agent { label 'built-in' }
             steps {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
-
                 script {
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                     MACHINE = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
-- 
GitLab


From 2b7e941b1f7f713151deead6cd5c95bd1a541ca7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 21:56:53 +0000
Subject: [PATCH 256/447] made machine global

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d044503db..df92e7d53 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,4 +1,5 @@
 def MACHINE = 'none'
+def machine = 'none'
 def HOMEgfs = 'initial value'
 
 pipeline {
@@ -16,7 +17,6 @@ pipeline {
             agent { label 'built-in' }
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                     MACHINE = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
@@ -40,6 +40,7 @@ pipeline {
             //}
             steps {
                 script {
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-- 
GitLab


From 38ad4b4c2fec5d2c93e28ffc038d2845f7c27c24 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 22:00:01 +0000
Subject: [PATCH 257/447] Label issue

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index df92e7d53..fc0c9fcc3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -115,7 +115,6 @@ pipeline {
                 if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) {
                    pullRequest.removeLabel("CI-${machine}-Running")
                 }
-                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Passed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
@@ -127,7 +126,6 @@ pipeline {
                 if(pullRequest.labels.contains("CI-${machine}-Running")) {
                    pullRequest.removeLabel("CI-${machine}-Running")
                 } 
-                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-- 
GitLab


From 9e5dacf57102fa65bd4f74cd5d18c63818abbe04 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 22:04:23 +0000
Subject: [PATCH 258/447] removed Configuiration class

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fc0c9fcc3..9fb7079f3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
             //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']]), [$class: 'JobLocalConfiguration', changeReasonComment: '']])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-- 
GitLab


From 2ef5a62152dc0e68dade518a1a81cbdfd8150370 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 22:40:02 +0000
Subject: [PATCH 259/447] added stub logic

---
 ci/scripts/run-check_ci_stub.sh | 4 +++-
 ci/scripts/utils/ci_utils.sh    | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index d98b87db2..48cf160d1 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -6,4 +6,6 @@ HOMEgfs=$1
 pslot=$2
 
 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
-exit 1
\ No newline at end of file
+if [[ ${pslot} ~= "C48_S2SW" ]]; then
+     exit 1
+fi
\ No newline at end of file
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index a0b45d5d5..0e12a814a 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -88,6 +88,8 @@ function get_pslot () {
 
 function create_experiment () {
 
+  exit 0
+
   local yaml_config="${1}"
   cd "${HOMEgfs}" || exit 1
   pr_sha=$(git rev-parse --short HEAD)
-- 
GitLab


From a2cef598627d89240439515325a36639111c77da Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 22:48:08 +0000
Subject: [PATCH 260/447] remove exit 0 on create exper wrapper because once
 worked

---
 ci/scripts/utils/ci_utils.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 0e12a814a..a0b45d5d5 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -88,8 +88,6 @@ function get_pslot () {
 
 function create_experiment () {
 
-  exit 0
-
   local yaml_config="${1}"
   cd "${HOMEgfs}" || exit 1
   pr_sha=$(git rev-parse --short HEAD)
-- 
GitLab


From 88799463b258a2cc0c3897cf759bae6fd6a5673c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Jan 2024 23:57:37 +0000
Subject: [PATCH 261/447] two orion fake local adds

---
 modulefiles/module_gwsetup.orion.lua | 17 +++++------------
 ush/module-setup.sh                  |  7 ++-----
 2 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua
index 93a59c8e5..984cf19b5 100644
--- a/modulefiles/module_gwsetup.orion.lua
+++ b/modulefiles/module_gwsetup.orion.lua
@@ -3,19 +3,12 @@ Load environment to run GFS workflow ci scripts on Orion
 ]])
 
 prepend_path("MODULEPATH", "/apps/modulefiles/core")
-load(pathJoin("contrib","0.1"))
-load(pathJoin("rocoto","1.3.3"))
-load(pathJoin("git","2.28.0"))
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+-- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake")
 
-local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
-local python_ver=os.getenv("python_ver") or "3.10.8"
-
-load(pathJoin("stack-intel", stack_intel_ver))
-load(pathJoin("python", python_ver))
-load("py-jinja2")
-load("py-pyyaml")
-load("py-numpy")
+load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj")
+load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4")
+load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip")
 
 whatis("Description: GFS run setup environment")
diff --git a/ush/module-setup.sh b/ush/module-setup.sh
index fd656966b..97df183a9 100755
--- a/ush/module-setup.sh
+++ b/ush/module-setup.sh
@@ -30,11 +30,8 @@ elif [[ ${MACHINE_ID} = hercules* ]] ; then
     set -u
 
 elif [[ ${MACHINE_ID} = orion* ]] ; then
-    # We are on Orion
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        source /apps/lmod/lmod/init/bash
-    fi
-    export LMOD_SYSTEM_DEFAULT_MODULES=contrib
+    # We are on Orion TMcG
+    source $(spack location -i lmod)/lmod/lmod/init/bash
     set +u
     module reset
     set -u
-- 
GitLab


From bf18bc80b6bc91e317b8b759b361fc7dc81d6ac3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 00:11:50 +0000
Subject: [PATCH 262/447] added my spak python 3

---
 modulefiles/module_gwsetup.orion.lua | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua
index 984cf19b5..03780b750 100644
--- a/modulefiles/module_gwsetup.orion.lua
+++ b/modulefiles/module_gwsetup.orion.lua
@@ -7,6 +7,7 @@ prepend_path("MODULEPATH", "/apps/modulefiles/core")
 -- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake")
 
+load ("python-3.9.12-gcc-7.3.1-cfrtozo")
 load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj")
 load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4")
 load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip")
-- 
GitLab


From 26a3512778bba9476ff7d9e66c11bcbf66dd28d2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 00:20:45 +0000
Subject: [PATCH 263/447] got =~ backwards

---
 ci/scripts/run-check_ci_stub.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index 48cf160d1..e01531d68 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -6,6 +6,6 @@ HOMEgfs=$1
 pslot=$2
 
 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
-if [[ ${pslot} ~= "C48_S2SW" ]]; then
+if [[ ${pslot} =~ "C48_S2SW" ]]; then
      exit 1
 fi
\ No newline at end of file
-- 
GitLab


From 77af83d336680a2d6361489615f37ea82065e3b3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 01:32:52 +0000
Subject: [PATCH 264/447] explitly remove existing experment before re-running
 create

---
 ci/scripts/utils/ci_utils.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index a0b45d5d5..22f4d496f 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -92,11 +92,17 @@ function create_experiment () {
   cd "${HOMEgfs}" || exit 1
   pr_sha=$(git rev-parse --short HEAD)
   case=$(basename "${yaml_config}" .yaml) || true
+  export pslot=${case}_${pr_sha}
   
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  export pslot=${case}_${pr_sha}
+  if [[ -d "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" ]]; then
+    echo "Removing existing experiment directory: ${HOMEgfs}/RUNDIRS/EXPDIR/${case}_${pr_sha}"
+    rm -rf "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}"
+    rm -rf "${HOMEgfs}/RUNDIRS/COMROT/${pslot}"
+  fi
+
   "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
-- 
GitLab


From b6ffee942da7afc244643a5da0232f25b77486f3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 01:38:41 +0000
Subject: [PATCH 265/447] hard coded HOMEgfs for testing

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9fb7079f3..875709bda 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,6 @@
 def MACHINE = 'none'
 def machine = 'none'
-def HOMEgfs = 'initial value'
+def HOMEgfs = '/home/tmcguinness/Jenkins/Hera/workspace/Global_PR-218'
 
 pipeline {
     agent { label 'built-in' }
@@ -34,7 +34,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "${MACHINE}-emc" }
+            agent { label "orion-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -71,7 +71,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "${MACHINE}-emc" }
+                agent { label "orion-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From 7c7fbbe324a95bcae4b2064772ee1d19cc411649 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 01:41:31 +0000
Subject: [PATCH 266/447] testing on node hera not orion

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 875709bda..eae8ec2ad 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -34,7 +34,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "orion-emc" }
+            agent { label "hera-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -71,7 +71,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "orion-emc" }
+                agent { label "hera-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From 0638b6df0667aa18ffeaa8dd6a36baba5dd8dc03 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 01:52:06 +0000
Subject: [PATCH 267/447] dump remove label constraint for now

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index eae8ec2ad..29c1036fa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
             steps {
                 script {
                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    pullRequest.removeLabel("CI-${machine}-Ready")
+                    //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
                     HOMEgfs = "${WORKSPACE}"
@@ -60,7 +60,7 @@ pipeline {
                         sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                    pullRequest.removeLabel("CI-${machine}-Building")
+                    //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
-- 
GitLab


From 6fc3581fe20beb24ec91a71fef6f99361763e90b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Jan 2024 02:14:04 +0000
Subject: [PATCH 268/447] RUNTESTS was wrong and yes works

---
 ci/scripts/utils/ci_utils.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 22f4d496f..9c9c5780e 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -97,12 +97,8 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  if [[ -d "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}" ]]; then
-    echo "Removing existing experiment directory: ${HOMEgfs}/RUNDIRS/EXPDIR/${case}_${pr_sha}"
-    rm -rf "${HOMEgfs}/RUNDIRS/EXPDIR/${pslot}"
-    rm -rf "${HOMEgfs}/RUNDIRS/COMROT/${pslot}"
-  fi
+  set -eux
 
-  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  yes | ${HOMEgfs}/workflow/create_experiment.py --yaml "${yaml_config}"
 
 }
-- 
GitLab


From 4e807cc2a8fc349c2f701980ee0e8f394343686f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 15:29:10 +0000
Subject: [PATCH 269/447] removed hardcoded machine label on nodes

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 29c1036fa..caa17fc1c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,6 @@
 def MACHINE = 'none'
 def machine = 'none'
-def HOMEgfs = '/home/tmcguinness/Jenkins/Hera/workspace/Global_PR-218'
+def HOMEgfs = 'none'
 
 pipeline {
     agent { label 'built-in' }
@@ -34,7 +34,7 @@ pipeline {
         }
 
         stage('Build') {
-            agent { label "hera-emc" }
+            agent { label "${MACHINE}-emc" }
             //when {
             //    expression { MACHINE != 'none' }
             //}
@@ -71,7 +71,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "hera-emc" }
+                agent { label "${machine}-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From b66b9bcaf77fc6368084c0695b0bf246e8765472 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 15:40:17 +0000
Subject: [PATCH 270/447] restore away from local build cludges

---
 modulefiles/module_gwsetup.orion.lua | 18 ++++++++++++------
 ush/module-setup.sh                  |  7 +++++--
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua
index 03780b750..93a59c8e5 100644
--- a/modulefiles/module_gwsetup.orion.lua
+++ b/modulefiles/module_gwsetup.orion.lua
@@ -3,13 +3,19 @@ Load environment to run GFS workflow ci scripts on Orion
 ]])
 
 prepend_path("MODULEPATH", "/apps/modulefiles/core")
+load(pathJoin("contrib","0.1"))
+load(pathJoin("rocoto","1.3.3"))
+load(pathJoin("git","2.28.0"))
 
--- prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/shared/spack/share/spack/modules/linux-amzn2-skylake")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
-load ("python-3.9.12-gcc-7.3.1-cfrtozo")
-load ("py-numpy-1.22.4-gcc-7.3.1-ylp7efj")
-load ("py-pyyaml-6.0-gcc-7.3.1-z36arc4")
-load ("py-jinja2-3.0.3-gcc-7.3.1-oegomip")
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
+local python_ver=os.getenv("python_ver") or "3.10.8"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
 
 whatis("Description: GFS run setup environment")
diff --git a/ush/module-setup.sh b/ush/module-setup.sh
index 97df183a9..fd656966b 100755
--- a/ush/module-setup.sh
+++ b/ush/module-setup.sh
@@ -30,8 +30,11 @@ elif [[ ${MACHINE_ID} = hercules* ]] ; then
     set -u
 
 elif [[ ${MACHINE_ID} = orion* ]] ; then
-    # We are on Orion TMcG
-    source $(spack location -i lmod)/lmod/lmod/init/bash
+    # We are on Orion
+    if ( ! eval module help > /dev/null 2>&1 ) ; then
+        source /apps/lmod/lmod/init/bash
+    fi
+    export LMOD_SYSTEM_DEFAULT_MODULES=contrib
     set +u
     module reset
     set -u
-- 
GitLab


From 29d790cb7a23f607591a9d9458b9db52eab37eca Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 15:53:24 +0000
Subject: [PATCH 271/447] some linting

---
 ci/scripts/run-check_ci_stub.sh | 2 +-
 ci/scripts/utils/ci_utils.sh    | 2 +-
 sorc/build_all_stub.sh          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index e01531d68..b58664992 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -5,7 +5,7 @@ set -eu
 HOMEgfs=$1
 pslot=$2
 
-echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> ${HOMEgfs}/ci/scripts/run-check_ci_stub.log
+echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log"
 if [[ ${pslot} =~ "C48_S2SW" ]]; then
      exit 1
 fi
\ No newline at end of file
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 9c9c5780e..600778db6 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -99,6 +99,6 @@ function create_experiment () {
 
   set -eux
 
-  yes | ${HOMEgfs}/workflow/create_experiment.py --yaml "${yaml_config}"
+  yes || true | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh
index 9acf802c4..a5ddf98d1 100755
--- a/sorc/build_all_stub.sh
+++ b/sorc/build_all_stub.sh
@@ -2,4 +2,4 @@
 
 set -eu
 
-echo "Ran build all stub in ${PWD}" > BUILD_STUB_RAN_$(date +%d%H%M)
\ No newline at end of file
+echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)"
\ No newline at end of file
-- 
GitLab


From 1b47f78c22acdb6475284b449722c49e4475a3d8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 15:54:37 +0000
Subject: [PATCH 272/447] some eols

---
 ci/scripts/run-check_ci_stub.sh | 2 +-
 sorc/build_all_stub.sh          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
index b58664992..23d3da837 100755
--- a/ci/scripts/run-check_ci_stub.sh
+++ b/ci/scripts/run-check_ci_stub.sh
@@ -8,4 +8,4 @@ pslot=$2
 echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log"
 if [[ ${pslot} =~ "C48_S2SW" ]]; then
      exit 1
-fi
\ No newline at end of file
+fi
diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh
index a5ddf98d1..6221b6bb5 100755
--- a/sorc/build_all_stub.sh
+++ b/sorc/build_all_stub.sh
@@ -2,4 +2,4 @@
 
 set -eu
 
-echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)"
\ No newline at end of file
+echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)"
-- 
GitLab


From a30a5553c60d8047f77b6b0960bd68a2f5ade7f4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 16:21:41 +0000
Subject: [PATCH 273/447] wrong MACHINE in node label

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index caa17fc1c..4832e1aee 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -71,7 +71,7 @@ pipeline {
             //    expression { MACHINE != 'none' }
             //}
             matrix {
-                agent { label "${machine}-emc" }
+                agent { label "${MACHINE}-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From ad1c71f7bffc43fe33055050de6d5fc9e8bd9b08 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 16:34:47 +0000
Subject: [PATCH 274/447] removed y pipe and added rm case in pipeline

---
 Jenkinsfile                  | 1 +
 ci/scripts/utils/ci_utils.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4832e1aee..b7be83407 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -85,6 +85,7 @@ pipeline {
                             ws(HOMEgfs) {
                                 script {
                                     env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
+                                    sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" )
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
                                 }
                             }
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 600778db6..a92ce66dc 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -99,6 +99,6 @@ function create_experiment () {
 
   set -eux
 
-  yes || true | "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
-- 
GitLab


From 65cc185e4b45fa22f190e45f6aa987f8e722bf84 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 16:39:56 +0000
Subject: [PATCH 275/447] removed DefaultSlave in parameters

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b7be83407..dc8d01e61 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
             //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['Hera-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-- 
GitLab


From 30dea74c6657400bd406f33a54c0c9e5aa0c95d6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 17:09:24 +0000
Subject: [PATCH 276/447] added rm COMROOT line in pipeline

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dc8d01e61..0d7fa4fe0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
             //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
@@ -86,7 +86,8 @@ pipeline {
                                 script {
                                     env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
                                     sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" )
-                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: false)
+                                    sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/COMROOT/${Case}_*" )
+                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 }
                             }
                         }
-- 
GitLab


From 8657be527cf7401d7fca63fcddcc0cd9e707d2a1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 17:33:55 +0000
Subject: [PATCH 277/447] try var in param

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0d7fa4fe0..bd008aa85 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
             //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['${machine}-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-- 
GitLab


From 995b437cf1a768da928a85c2ac45485b17176efd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 17:43:50 +0000
Subject: [PATCH 278/447] trying to add more choices

---
 Jenkinsfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bd008aa85..42e29d474 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,8 @@ pipeline {
             //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['${machine}-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+                                        [choice(choices: ['C48_ATM', 'C48_S2SW', 'C96_atm3DVar', 'C48_S2SWA_gefs'], name: 'Cases')] ])])
                     //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
@@ -53,8 +54,8 @@ pipeline {
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
-                        //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                        sh( script: "sorc/build_all_stub.sh" )
+                        sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                        //sh( script: "sorc/build_all_stub.sh" )
                         sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true)
                         sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true)
                         sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true)
@@ -98,8 +99,8 @@ pipeline {
                                 script {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
-- 
GitLab


From c2a9c865d497601e68d1a1f78b6b74468df228ab Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 17:48:38 +0000
Subject: [PATCH 279/447] add back real build run

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 42e29d474..17b449886 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,9 +35,9 @@ pipeline {
 
         stage('Build') {
             agent { label "${MACHINE}-emc" }
-            //when {
-            //    expression { MACHINE != 'none' }
-            //}
+            when {
+                expression { MACHINE != 'none' }
+            }
             steps {
                 script {
                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
-- 
GitLab


From 49a7f716e7ac68577871a011577fb4fee62c1c3b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 18:09:15 +0000
Subject: [PATCH 280/447] revert

---
 Jenkinsfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 17b449886..cfc820c0f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,13 +35,12 @@ pipeline {
 
         stage('Build') {
             agent { label "${MACHINE}-emc" }
-            when {
-                expression { MACHINE != 'none' }
-            }
+            //when {
+            //    expression { MACHINE != 'none' }
+            //}
             steps {
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
-                                        [choice(choices: ['C48_ATM', 'C48_S2SW', 'C96_atm3DVar', 'C48_S2SWA_gefs'], name: 'Cases')] ])])
+                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     //pullRequest.removeLabel("CI-${machine}-Ready")
                     pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-- 
GitLab


From bfb1ef49edc20d9ba6cda37650276d69d8771fcb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:15:57 +0000
Subject: [PATCH 281/447] ready to start testing stub pipeline with system
 build

---
 Jenkinsfile                  | 57 ++++++++++++++++++++----------------
 ci/scripts/utils/ci_utils.sh |  4 +--
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index cfc820c0f..707170368 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -18,6 +18,7 @@ pipeline {
             steps {
                 script {
                     MACHINE = 'none'
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
@@ -29,37 +30,42 @@ pipeline {
                         }
                     }
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                    HOME = "${WORKSPACE}"
                 }
             }
         }
-
-        stage('Build') {
+        matrix {
             agent { label "${MACHINE}-emc" }
-            //when {
-            //    expression { MACHINE != 'none' }
-            //}
+            axes {
+                axes { name "system"
+                   values "gfs", "gefs"}
+            }  
+        }
+        stage('Build') {
             steps {
+                ws( "${HOME}/${system}")
                 script {
-                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    //pullRequest.removeLabel("CI-${machine}-Ready")
-                    pullRequest.addLabel("CI-${machine}-Building")
                     checkout scm
-                    HOMEgfs = "${WORKSPACE}"
-                }
-                script {
+                    HOMEgfs = "${HOME}/${system}"
                     env.MACHINE_ID = MACHINE
-                    if (fileExists("${HOMEgfs}/sorc/BUILT_sema")) {
+                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                         HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
-                        sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                        //sh( script: "sorc/build_all_stub.sh" )
-                        sh( script: "rm -Rf ${WORKSPACE}/RUNTESTS", returnStatus: true)
-                        sh( script: "mkdir -p ${WORKSPACE}/RUNTESTS", returnStatus: true)
-                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_sema", returnStatus: true)
+                        if (system == "gfs") {
+                            //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                            sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                        }
+                        else if (system == "gefs") {
+                            //sh( script: "sorc/build_all.sh -guw", returnStatus: false)
+                            sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                        }
+                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
@@ -75,21 +81,20 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_ATM", "C48_S2SW"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
                 stages {
                     stage('Create Experiment') {
                         steps {
-                            ws(HOMEgfs) {
                                 script {
-                                    env.RUNTESTS = "${HOMEgfs}/RUNTESTS"
-                                    sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/EXPDIR/${Case}_*" )
-                                    sh( script: "rm -Rf ${HOMEgfs}/RUNTESTS/COMROOT/${Case}_*" )
+                                    HOMEgfs = "${HOME}/gfs"
+                                    env.RUNTESTS = "${HOME}/RUNTESTS"
+                                    env.HOME = HOME
+                                    sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" )
+                                    sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" )
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 }
-                            }
                         }
                     }
                     stage('Run Experiments') {
@@ -98,8 +103,8 @@ pipeline {
                                 script {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
+                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                             }
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index a92ce66dc..f71221ccc 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -97,8 +97,8 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  set -eux
+  system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
 
-  "${HOMEgfs}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  "${HOME}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
-- 
GitLab


From a2e8f076d9d017020fd0d9c138a5854192f9c034 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:18:44 +0000
Subject: [PATCH 282/447] wrap a stages around build matrix stage

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 707170368..159edf889 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,6 +41,7 @@ pipeline {
                    values "gfs", "gefs"}
             }  
         }
+        stages {
         stage('Build') {
             steps {
                 ws( "${HOME}/${system}")
@@ -71,6 +72,7 @@ pipeline {
                 }
             }
         }
+        }
 
         stage('Run Tests') {
             //when {
-- 
GitLab


From d1e23592349144d5c858a6d2fd33abe1986b7460 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:22:25 +0000
Subject: [PATCH 283/447] wrap matrix around build stage

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 159edf889..d78792ab3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,8 +40,6 @@ pipeline {
                 axes { name "system"
                    values "gfs", "gefs"}
             }  
-        }
-        stages {
         stage('Build') {
             steps {
                 ws( "${HOME}/${system}")
-- 
GitLab


From 54baf54a4b2ac642282370cd55927d9f5b930fd5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:30:04 +0000
Subject: [PATCH 284/447] needed axis not two axes

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d78792ab3..9dede9d82 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -37,7 +37,7 @@ pipeline {
         matrix {
             agent { label "${MACHINE}-emc" }
             axes {
-                axes { name "system"
+                axis { name "system"
                    values "gfs", "gefs"}
             }  
         stage('Build') {
-- 
GitLab


From 04288ce646ce5f6c659fb2c0b3db3046451972da Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:34:31 +0000
Subject: [PATCH 285/447] put matrix wrap in buid stage

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9dede9d82..3870095aa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -34,13 +34,13 @@ pipeline {
                 }
             }
         }
+        stage('Build') {
         matrix {
             agent { label "${MACHINE}-emc" }
             axes {
                 axis { name "system"
                    values "gfs", "gefs"}
             }  
-        stage('Build') {
             steps {
                 ws( "${HOME}/${system}")
                 script {
-- 
GitLab


From b8b8070b11aa6404dd03225f40c55f5704779d63 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:38:01 +0000
Subject: [PATCH 286/447] still trying to get matrix hierachey to work

---
 Jenkinsfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3870095aa..24d77c29b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,9 @@ pipeline {
             axes {
                 axis { name "system"
                    values "gfs", "gefs"}
-            }  
+            }
+            stages {
+            stage("build ${system}") {
             steps {
                 ws( "${HOME}/${system}")
                 script {
@@ -69,6 +71,8 @@ pipeline {
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
+            }
+            }
         }
         }
 
-- 
GitLab


From dcf7a87c50249540b70047696056232b70a3f36e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:39:29 +0000
Subject: [PATCH 287/447] system literal literal

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 24d77c29b..7e66888a9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                    values "gfs", "gefs"}
             }
             stages {
-            stage("build ${system}") {
+            stage("build system") {
             steps {
                 ws( "${HOME}/${system}")
                 script {
-- 
GitLab


From 686be118a12284f0b564df389cd505cf485f5ce4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 20:52:18 +0000
Subject: [PATCH 288/447] ws needed body

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7e66888a9..dd5dc9de8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,7 +44,7 @@ pipeline {
             stages {
             stage("build system") {
             steps {
-                ws( "${HOME}/${system}")
+                ws( "${HOME}/${system}") {
                 script {
                     checkout scm
                     HOMEgfs = "${HOME}/${system}"
@@ -70,6 +70,7 @@ pipeline {
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
+                }
             }
             }
             }
-- 
GitLab


From 81380a850e5491a14dd354f06f7e3761f02c1ae8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 22:06:49 +0000
Subject: [PATCH 289/447] remove workspaces

---
 Jenkinsfile | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dd5dc9de8..0a6cbe666 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,13 +44,14 @@ pipeline {
             stages {
             stage("build system") {
             steps {
-                ws( "${HOME}/${system}") {
                 script {
+                    HOMEgfs = "${HOME}/HOMEgfs/${system}"
+                    sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                    dir(HOMEgfs)
                     checkout scm
-                    HOMEgfs = "${HOME}/${system}"
                     env.MACHINE_ID = MACHINE
-                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-                        HOMEgfs = sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim()
+                    if (fileExists("sorc/BUILT_semaphor")) {
+                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
@@ -70,7 +71,6 @@ pipeline {
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
-                }
             }
             }
             }
@@ -104,15 +104,14 @@ pipeline {
                     }
                     stage('Run Experiments') {
                         steps {
-                            ws(HOMEgfs) {
                                 script {
+                                    HOMEgfs = "${HOME}/gfs"
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
                                     sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
-                            }
                         }
                     }
                 }
-- 
GitLab


From e0a420fca038fb86d3bb5221d01c1dde8484b94b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 22:53:07 +0000
Subject: [PATCH 290/447] WORKSPACE git just outside of build matrix in node

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0a6cbe666..b2f727535 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -30,13 +30,13 @@ pipeline {
                         }
                     }
                     machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                    HOME = "${WORKSPACE}"
                 }
             }
         }
         stage('Build') {
         matrix {
             agent { label "${MACHINE}-emc" }
+            script { TESTDIR = "${WORKSPACE}/TESTDIR" }
             axes {
                 axis { name "system"
                    values "gfs", "gefs"}
@@ -45,7 +45,7 @@ pipeline {
             stage("build system") {
             steps {
                 script {
-                    HOMEgfs = "${HOME}/HOMEgfs/${system}"
+                    HOMEgfs = "${HOME}/${system}"
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs)
                     checkout scm
-- 
GitLab


From d9303a2f54746dd812f20fad1c444826cf5a0e86 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 22:59:30 +0000
Subject: [PATCH 291/447] Added TESTDIR

---
 Jenkinsfile | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b2f727535..82d0e1405 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -18,7 +18,6 @@ pipeline {
             steps {
                 script {
                     MACHINE = 'none'
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
@@ -45,7 +44,9 @@ pipeline {
             stage("build system") {
             steps {
                 script {
-                    HOMEgfs = "${HOME}/${system}"
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                    TESTDIR = "${WORKSPACE}/TESTDIR"
+                    HOMEgfs = "${TESTDIR}/${system}"
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs)
                     checkout scm
@@ -66,7 +67,7 @@ pipeline {
                         sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                    sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true)
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
@@ -93,9 +94,8 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                                 script {
-                                    HOMEgfs = "${HOME}/gfs"
-                                    env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    env.HOME = HOME
+                                    env.RUNTESTS = "${TESTDIR}/RUNTESTS"
+                                    env.HOME = HOMEgfs
                                     sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" )
                                     sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" )
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
@@ -105,7 +105,6 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                                 script {
-                                    HOMEgfs = "${HOME}/gfs"
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-- 
GitLab


From 9b28ba32cfb8c543730655e0e6d517fd98be7965 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 23:00:58 +0000
Subject: [PATCH 292/447] deleted old script line

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 82d0e1405..a6491ec4a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,7 +35,6 @@ pipeline {
         stage('Build') {
         matrix {
             agent { label "${MACHINE}-emc" }
-            script { TESTDIR = "${WORKSPACE}/TESTDIR" }
             axes {
                 axis { name "system"
                    values "gfs", "gefs"}
-- 
GitLab


From a6e08390e9a058ef91667633852f5473aed97a71 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 23:06:10 +0000
Subject: [PATCH 293/447] dir step needed bodey

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a6491ec4a..5e4fc1293 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
                     TESTDIR = "${WORKSPACE}/TESTDIR"
                     HOMEgfs = "${TESTDIR}/${system}"
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                    dir(HOMEgfs)
+                    dir(HOMEgfs) {
                     checkout scm
                     env.MACHINE_ID = MACHINE
                     if (fileExists("sorc/BUILT_semaphor")) {
@@ -70,6 +70,7 @@ pipeline {
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
+                    }
                 }
             }
             }
-- 
GitLab


From 6687bd54623745035cb3b2a7a58253360ae59bba Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 22 Jan 2024 23:27:03 +0000
Subject: [PATCH 294/447] created dedicated stage for getting global workspace
 for the PR

---
 Jenkinsfile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5e4fc1293..e4e165d0d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,7 @@
 def MACHINE = 'none'
 def machine = 'none'
 def HOMEgfs = 'none'
+def TESTDIR = 'none'
 
 pipeline {
     agent { label 'built-in' }
@@ -32,6 +33,16 @@ pipeline {
                 }
             }
         }
+
+        stage('Get Common Workspace') {
+            agent { label "${MACHINE}-emc" }
+            steps ( timeout(time: 1, unit: 'HOURS') ) {
+                script {
+                    TESTDIR = "${WORKSPACE}/TESTDIR"
+                }
+            }
+        }
+
         stage('Build') {
         matrix {
             agent { label "${MACHINE}-emc" }
@@ -44,7 +55,6 @@ pipeline {
             steps {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    TESTDIR = "${WORKSPACE}/TESTDIR"
                     HOMEgfs = "${TESTDIR}/${system}"
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs) {
-- 
GitLab


From 118d43d2a82c1fab5a5066fc831fdc1022578328 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 00:16:21 +0000
Subject: [PATCH 295/447] made HOMEgfs local in matrix and removed from global

---
 Jenkinsfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e4e165d0d..6b4e0b43b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,5 @@
 def MACHINE = 'none'
 def machine = 'none'
-def HOMEgfs = 'none'
 def TESTDIR = 'none'
 
 pipeline {
@@ -54,14 +53,14 @@ pipeline {
             stage("build system") {
             steps {
                 script {
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['Hera-EMC','Orion-EMC'], name: 'EMC RDHPCS', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    HOMEgfs = "${TESTDIR}/${system}"
+                    def HOMEgfs = "${TESTDIR}/${system}"
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs) {
                     checkout scm
                     env.MACHINE_ID = MACHINE
                     if (fileExists("sorc/BUILT_semaphor")) {
-                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_sema", returnStdout: true).trim()
+                        sh( script: "cat sorc/BUILT_sema", returnStdout: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
@@ -73,7 +72,7 @@ pipeline {
                             //sh( script: "sorc/build_all.sh -guw", returnStatus: false)
                             sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
-                        sh( script: "echo ${HOMEgfs} > ${HOMEgfs}/sorc/BUILT_semaphor", returnStatus: true)
+                        sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true)
-- 
GitLab


From e601257c6b4a47b8a1d179f996b71863e8066277 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 01:19:08 +0000
Subject: [PATCH 296/447] global HOME TESTDIR

---
 Jenkinsfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6b4e0b43b..e06e0f967 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,7 @@
 def MACHINE = 'none'
 def machine = 'none'
 def TESTDIR = 'none'
+def HOME = 'none'
 
 pipeline {
     agent { label 'built-in' }
@@ -37,7 +38,7 @@ pipeline {
             agent { label "${MACHINE}-emc" }
             steps ( timeout(time: 1, unit: 'HOURS') ) {
                 script {
-                    TESTDIR = "${WORKSPACE}/TESTDIR"
+                    HOME = "${WORKSPACE}/TESTDIR"
                 }
             }
         }
@@ -53,14 +54,14 @@ pipeline {
             stage("build system") {
             steps {
                 script {
-                    def HOMEgfs = "${TESTDIR}/${system}"
+                    def HOMEgfs = "${HOME}/${system}"
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs) {
                     checkout scm
                     env.MACHINE_ID = MACHINE
                     if (fileExists("sorc/BUILT_semaphor")) {
-                        sh( script: "cat sorc/BUILT_sema", returnStdout: true).trim()
+                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                     }
                     else {
@@ -104,6 +105,7 @@ pipeline {
                         steps {
                                 script {
                                     env.RUNTESTS = "${TESTDIR}/RUNTESTS"
+                                    def HOMEgfs = "${HOME}/TESTDIR/gfs"
                                     env.HOME = HOMEgfs
                                     sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" )
                                     sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" )
-- 
GitLab


From c422f617a88641c16bd361079f3d399384decfd8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 01:52:40 +0000
Subject: [PATCH 297/447] added local HOMEgfs and global HOME in Run
 Experiments

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e06e0f967..41fc6181b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,5 @@
 def MACHINE = 'none'
 def machine = 'none'
-def TESTDIR = 'none'
 def HOME = 'none'
 
 pipeline {
@@ -104,8 +103,8 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                                 script {
-                                    env.RUNTESTS = "${TESTDIR}/RUNTESTS"
-                                    def HOMEgfs = "${HOME}/TESTDIR/gfs"
+                                    env.RUNTESTS = "${HOME}/RUNTESTS"
+                                    def HOMEgfs = "${HOME}/gfs"
                                     env.HOME = HOMEgfs
                                     sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" )
                                     sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" )
@@ -116,10 +115,11 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                                 script {
-                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOMEgfs}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                    def HOMEgfs = "${HOME}/gfs"
+                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOMEgfs} ${pslot}")
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                         }
-- 
GitLab


From 86f2c903b0e1941f06c2beb29f254446656589ec Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 02:05:26 +0000
Subject: [PATCH 298/447] updated TESTDIR to global HOME

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 41fc6181b..9ecf37a7f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -75,7 +75,7 @@ pipeline {
                         sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                    sh( script: "mkdir -p ${TESTDIR}/RUNTESTS", returnStatus: true)
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
                     //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
-- 
GitLab


From 45601f5be369cf7565b174e541478d54cd3ca3f0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 02:34:27 +0000
Subject: [PATCH 299/447] remved stubs to run on Orion

---
 Jenkinsfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9ecf37a7f..8ded06712 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -65,12 +65,12 @@ pipeline {
                     }
                     else {
                         if (system == "gfs") {
-                            //sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                            sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                            sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                            //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         else if (system == "gefs") {
-                            //sh( script: "sorc/build_all.sh -guw", returnStatus: false)
-                            sh( script: "sorc/build_all_stub.sh", returnStatus: false)
+                            sh( script: "sorc/build_all.sh -guw", returnStatus: false)
+                            //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                     }
@@ -118,8 +118,8 @@ pipeline {
                                     def HOMEgfs = "${HOME}/gfs"
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}")
+                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                         }
-- 
GitLab


From 3146a39d1619c18bb587c1b9ae80f8ceceb154fe Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 02:46:53 +0000
Subject: [PATCH 300/447] need updated buid all args for gefs

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ded06712..9f216031c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -69,7 +69,7 @@ pipeline {
                             //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         else if (system == "gefs") {
-                            sh( script: "sorc/build_all.sh -guw", returnStatus: false)
+                            sh( script: "sorc/build_all.sh -gu -a UFS_app", returnStatus: false)
                             //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
-- 
GitLab


From 343020c9f6393380fc1e350ca7024f437dbd0963 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 04:20:55 +0000
Subject: [PATCH 301/447] use old build all args for both systems

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9f216031c..450608b48 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -69,7 +69,7 @@ pipeline {
                             //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         else if (system == "gefs") {
-                            sh( script: "sorc/build_all.sh -gu -a UFS_app", returnStatus: false)
+                            sh( script: "sorc/build_all.sh -gu", returnStatus: false)
                             //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
-- 
GitLab


From 508e354e6783ac1f11f10f316ab246369e2ffbc0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 14:33:19 +0000
Subject: [PATCH 302/447] removed stubs add all nodes with restart selector and
 put back label rm

---
 Jenkinsfile | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 450608b48..4ffc1dc0c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,7 +8,7 @@ pipeline {
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
-        buildDiscarder(logRotator(numToKeepStr: '2'))
+        // buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
     stages {
@@ -54,7 +54,7 @@ pipeline {
             steps {
                 script {
                     def HOMEgfs = "${HOME}/${system}"
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['ALL (no restriction)', 'built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                     dir(HOMEgfs) {
                     checkout scm
@@ -66,18 +66,17 @@ pipeline {
                     else {
                         if (system == "gfs") {
                             sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                            //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         else if (system == "gefs") {
+                            // TODO: need to add gefs build arguments from a yaml file
                             sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                            //sh( script: "sorc/build_all_stub.sh", returnStatus: false)
                         }
                         sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                     }
                     sh( script: "sorc/link_workflow.sh", returnStatus: false)
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
-                    //pullRequest.removeLabel("CI-${machine}-Building")
+                    pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                     }
                 }
@@ -88,9 +87,6 @@ pipeline {
         }
 
         stage('Run Tests') {
-            //when {
-            //    expression { MACHINE != 'none' }
-            //}
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
@@ -119,7 +115,6 @@ pipeline {
                                     pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                     pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                     sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
-                                    //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci_stub.sh ${HOME} ${pslot}")
                                     pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                }
                         }
-- 
GitLab


From 2b684c29d90623fd8ae1f76cb1bd749f2193af0e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 14:37:02 +0000
Subject: [PATCH 303/447] remove Running label without check

---
 Jenkinsfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4ffc1dc0c..d7992ad8f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -128,9 +128,11 @@ pipeline {
     post {
         success {
             script {
-                if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) {
-                   pullRequest.removeLabel("CI-${machine}-Running")
-                }
+                //if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) {
+                //   pullRequest.removeLabel("CI-${machine}-Running")
+                //}
+                // TODO: contains mehthod does not work
+                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Passed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
@@ -139,9 +141,7 @@ pipeline {
         }
         failure {
             script {
-                if(pullRequest.labels.contains("CI-${machine}-Running")) {
-                   pullRequest.removeLabel("CI-${machine}-Running")
-                } 
+                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-- 
GitLab


From 6775d4988357f0e851c0b515ea4e0338145f8bf8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 20:32:49 +0000
Subject: [PATCH 304/447] add dir into sorc

---
 Jenkinsfile | 107 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 66 insertions(+), 41 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d7992ad8f..225d3530a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -43,47 +43,61 @@ pipeline {
         }
 
         stage('Build') {
-        matrix {
-            agent { label "${MACHINE}-emc" }
-            axes {
-                axis { name "system"
-                   values "gfs", "gefs"}
-            }
-            stages {
-            stage("build system") {
-            steps {
-                script {
-                    def HOMEgfs = "${HOME}/${system}"
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                    dir(HOMEgfs) {
-                    checkout scm
-                    env.MACHINE_ID = MACHINE
-                    if (fileExists("sorc/BUILT_semaphor")) {
-                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                axes {
+                    axis { 
+                        name "system"
+                        values "gfs", "gefs"
                     }
-                    else {
-                        if (system == "gfs") {
-                            sh( script: "sorc/build_all.sh -gu", returnStatus: false)
-                        }
-                        else if (system == "gefs") {
-                            // TODO: need to add gefs build arguments from a yaml file
-                            sh( script: "sorc/build_all.sh -gu", returnStatus: false)
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                dir(HOMEgfs) {
+                                    checkout scm
+                                    env.MACHINE_ID = MACHINE
+                                    if (fileExists("sorc/BUILT_semaphor")) {
+                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                    } else {
+                                        if (system == "gfs") {
+                                            dir("sorc") {
+                                                sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) 
+                                                sh( script: "build_all.sh -gu", returnStatus: false)
+                                            }
+                                        } else if (system == "gefs") {
+                                            // TODO: need to add gefs build arguments from a yaml file
+                                            dir("sorc") {
+                                                sh( script: "echo $PWD", returnStatus: true) 
+                                                sh( script: "build_all.sh -gu", returnStatus: false)
+                                            }
+                                        }
+                                        sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
+                                    }
+                                    sh( script: "sorc/link_workflow.sh", returnStatus: false)
+                                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
+                                    pullRequest.removeLabel("CI-${machine}-Building")
+                                    pullRequest.addLabel("CI-${machine}-Running")
+                                }
+                            }
                         }
-                        sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                     }
-                    sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
+                }
+                script {
                     pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
-                    }
                 }
             }
+            script {
+                    pullRequest.removeLabel("CI-${machine}-Building")
+                    pullRequest.addLabel("CI-${machine}-Running")
             }
-            }
-        }
         }
 
         stage('Run Tests') {
@@ -110,18 +124,29 @@ pipeline {
                     }
                     stage('Run Experiments') {
                         steps {
-                                script {
-                                    def HOMEgfs = "${HOME}/gfs"
-                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                    pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                                    sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
-                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
-                               }
+                            script {
+                                def HOMEgfs = "${HOME}/gfs"
+                                pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                            try {
+                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                            } catch (Exception e) {
+                                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                                    fileContent.eachLine { line ->
+                                        archiveArtifacts artifacts: "${line}", fingerprint: true
+                                    }
+                                }
+                            }
+                                    pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
+                                    error("Failed to run experiments ${Case} on ${machine}")
+                            }
                         }
                     }
                 }
             }
-       }
+        }
 
     }
 
-- 
GitLab


From 785608124e0f5bc625bfde82bdb4863adffa353f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 20:51:58 +0000
Subject: [PATCH 305/447] moved mkdir on RUNTESTS into top stage in Create
 Experments

---
 Jenkinsfile | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 225d3530a..87f9e3593 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -69,38 +69,36 @@ pipeline {
                                             dir("sorc") {
                                                 sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) 
                                                 sh( script: "build_all.sh -gu", returnStatus: false)
+                                                sh( script: "link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
                                             // TODO: need to add gefs build arguments from a yaml file
                                             dir("sorc") {
                                                 sh( script: "echo $PWD", returnStatus: true) 
                                                 sh( script: "build_all.sh -gu", returnStatus: false)
+                                                sh( script: "link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         }
-                                        sh( script: "echo ${HOMEgfs} > sorc/BUILT_semaphor", returnStatus: true)
                                     }
-                                    sh( script: "sorc/link_workflow.sh", returnStatus: false)
-                                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
-                                    pullRequest.removeLabel("CI-${machine}-Building")
-                                    pullRequest.addLabel("CI-${machine}-Running")
                                 }
                             }
                         }
                     }
                 }
+            }
+        }
+
+        stage('Run Tests') {
+            steps {
                 script {
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
                     pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
-            script {
-                    pullRequest.removeLabel("CI-${machine}-Building")
-                    pullRequest.addLabel("CI-${machine}-Running")
-            }
-        }
-
-        stage('Run Tests') {
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
-- 
GitLab


From 75bccbd7dd0bc7155795fc3d93a9f8fb37412cb4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 21:17:04 +0000
Subject: [PATCH 306/447] added entire stage to mkdir TESTDIR and update labels

---
 Jenkinsfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 87f9e3593..be6d3659b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -90,7 +90,7 @@ pipeline {
             }
         }
 
-        stage('Run Tests') {
+        stage('Setup RUNTESTS') {
             steps {
                 script {
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
@@ -99,6 +99,10 @@ pipeline {
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
+
+        }
+
+        stage('Run Tests') {
             matrix {
                 agent { label "${MACHINE}-emc" }
                 axes {
-- 
GitLab


From b2b52432b37b44712385c410e4cfe902b71315f8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 21:29:28 +0000
Subject: [PATCH 307/447] added full path for dir() into sorc

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index be6d3659b..1193222d6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,16 +66,16 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         if (system == "gfs") {
-                                            dir("sorc") {
-                                                sh( script: "echo $PWD;which ls;ls --version", returnStatus: true) 
+                                            dir("${HOMEgfs}/sorc") {
+                                                sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) 
                                                 sh( script: "build_all.sh -gu", returnStatus: false)
                                                 sh( script: "link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
                                             // TODO: need to add gefs build arguments from a yaml file
-                                            dir("sorc") {
-                                                sh( script: "echo $PWD", returnStatus: true) 
+                                            dir("${HOMEgfs}/sorc") {
+                                                sh( script: "echo 'In gefs block in dir $PWD'", returnStatus: true) 
                                                 sh( script: "build_all.sh -gu", returnStatus: false)
                                                 sh( script: "link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-- 
GitLab


From 5965d86b2ba04346c7b2811b693d3a67b47c129e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 21:30:14 +0000
Subject: [PATCH 308/447] reduced save to three

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1193222d6..c20506b96 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,7 +8,7 @@ pipeline {
     options {
         disableConcurrentBuilds(abortPrevious: true)
         skipDefaultCheckout(true)
-        // buildDiscarder(logRotator(numToKeepStr: '2'))
+        buildDiscarder(logRotator(numToKeepStr: '3'))
     }
 
     stages {
-- 
GitLab


From e22dffaa134005b8de22cc93eb9b885897c8bd7e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 22:05:27 +0000
Subject: [PATCH 309/447] can not figure out why dir() is not working

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index c20506b96..a9263a32a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,6 +66,8 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         if (system == "gfs") {
+                                            sh( script: "echo 'What is going on here: ${HOMEgfs}/sorc'", returnStatus: true)
+                                            sh( script: "ls -lrtha ${HOMEgfs}/sorc'", returnStatus: true)
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) 
                                                 sh( script: "build_all.sh -gu", returnStatus: false)
-- 
GitLab


From b394c3a520dea2964c67bf50bcc78cd7889d4389 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 23 Jan 2024 22:31:34 +0000
Subject: [PATCH 310/447] added a few more degubs and . for pwd path

---
 Jenkinsfile | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a9263a32a..532b98a47 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,20 +66,21 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         if (system == "gfs") {
-                                            sh( script: "echo 'What is going on here: ${HOMEgfs}/sorc'", returnStatus: true)
-                                            sh( script: "ls -lrtha ${HOMEgfs}/sorc'", returnStatus: true)
+                                            sh( script: "echo 'What is going on here in gfs block: ${HOMEgfs}/sorc'", returnStatus: true)
+                                            sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true)
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version", returnStatus: true) 
-                                                sh( script: "build_all.sh -gu", returnStatus: false)
-                                                sh( script: "link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) 
+                                                sh( script: "./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
+                                            sh( script: "echo 'What is going on here in gefs: ${HOMEgfs}/sorc'", returnStatus: true)
                                             // TODO: need to add gefs build arguments from a yaml file
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "echo 'In gefs block in dir $PWD'", returnStatus: true) 
-                                                sh( script: "build_all.sh -gu", returnStatus: false)
-                                                sh( script: "link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo 'In gefs block in dir $PWD';ls -l", returnStatus: true) 
+                                                sh( script: "./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         }
-- 
GitLab


From 9b02f36849283c0be0bf2a5bed73cd6777d4881d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 25 Jan 2024 16:01:54 +0000
Subject: [PATCH 311/447] added an explicit cd into the gfs/sorc dir to run
 build_all

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 532b98a47..7fc00f355 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -70,7 +70,7 @@ pipeline {
                                             sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true)
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) 
-                                                sh( script: "./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "cd ${HOMEgfs}/sorc;echo "PWD:${PWD};./build_all.sh -gu", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
-- 
GitLab


From c7053d14e3ce13ee6a4b0fa4393c0b1d8d1a1809 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 25 Jan 2024 16:09:46 +0000
Subject: [PATCH 312/447] fixed quote on echo PWD on script sh command line

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7fc00f355..efa9cbb56 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -70,7 +70,7 @@ pipeline {
                                             sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true)
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) 
-                                                sh( script: "cd ${HOMEgfs}/sorc;echo "PWD:${PWD};./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "cd ${HOMEgfs}/sorc;echo 'PWD:${PWD}';./build_all.sh -gu", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
-- 
GitLab


From 457b3d7b7e818e73a4bdd53edb558803e05df02d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 25 Jan 2024 19:35:31 +0000
Subject: [PATCH 313/447] added explicit git command for updating submoules
 recursivly, scm was apearing to skip gdas submodules

---
 Jenkinsfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index efa9cbb56..3474d8136 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -65,20 +65,16 @@ pipeline {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         if (system == "gfs") {
-                                            sh( script: "echo 'What is going on here in gfs block: ${HOMEgfs}/sorc'", returnStatus: true)
-                                            sh( script: "ls -lrtha ${HOMEgfs}/sorc", returnStatus: true)
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "echo 'In gfs block in dir $PWD';which ls;ls --version;ls", returnStatus: true) 
-                                                sh( script: "cd ${HOMEgfs}/sorc;echo 'PWD:${PWD}';./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
-                                            sh( script: "echo 'What is going on here in gefs: ${HOMEgfs}/sorc'", returnStatus: true)
                                             // TODO: need to add gefs build arguments from a yaml file
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "echo 'In gefs block in dir $PWD';ls -l", returnStatus: true) 
                                                 sh( script: "./build_all.sh -gu", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-- 
GitLab


From 6acc9180c5b1ec9950f794b28460a939f3399d86 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 26 Jan 2024 16:32:46 +0000
Subject: [PATCH 314/447] added max jobs on build all to 4

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3474d8136..530ce7d16 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,7 +68,7 @@ pipeline {
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         if (system == "gfs") {
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
-- 
GitLab


From 258e9c7b4d9debacbca348e521b549b7f6f51d2d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 26 Jan 2024 22:21:50 +0000
Subject: [PATCH 315/447] updated pipline with node on setup RUNDIR and removed
 system in run experment wrapper

---
 Jenkinsfile                  | 5 +++--
 ci/scripts/utils/ci_utils.sh | 4 +---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 530ce7d16..556ed91a9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -90,11 +90,12 @@ pipeline {
         }
 
         stage('Setup RUNTESTS') {
+            agent { label "${MACHINE}-emc" }
             steps {
                 script {
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
-                    pullRequest.removeLabel("CI-${machine}-Building")
+                    //pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
@@ -130,7 +131,7 @@ pipeline {
                                 pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             try {
-                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             } catch (Exception e) {
                                 if (fileExists('${HOME}/RUNTESTS/ci.log')) {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index f71221ccc..6b9312257 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -97,8 +97,6 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
-
-  "${HOME}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  "${HOME}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
-- 
GitLab


From 960e0fe2e42ca67ed2089b905d79193b2fede056 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 26 Jan 2024 22:53:32 +0000
Subject: [PATCH 316/447] updated path of HOMEgfs in run check scripts using
 gfs as default

---
 ci/scripts/run-check_ci.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 37761573d..f98f43446 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -21,7 +21,9 @@ pslot=${2:-${pslot:-?}}        # Name of the experiment being tested by this scr
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
-HOMEgfs="${TEST_DIR}"
+# Two system build directories created at build time gfs, and gdas
+# TODO: Make this configurable (for now all scripts run from gfs for CI at runtime)
+HOMEgfs="${TEST_DIR}/gfs"
 RUNTESTS="${TEST_DIR}/RUNTESTS"
 
 # Source modules and setup logging
-- 
GitLab


From 0d09e5c15b5e7096ac1e61a4bc4d039e32f3b9fe Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 26 Jan 2024 22:57:05 +0000
Subject: [PATCH 317/447] use HOME to point to RUNDIRS for run-check

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 556ed91a9..20c314e7e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -131,7 +131,7 @@ pipeline {
                                 pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             try {
-                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOMEgfs} ${pslot}", returnStatus: false)
+                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             } catch (Exception e) {
                                 if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-- 
GitLab


From 4eb89e24b46862783dac1e8c965340e11362cb25 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 27 Jan 2024 07:16:55 +0000
Subject: [PATCH 318/447] add label updates back move archive to post

---
 Jenkinsfile | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 20c314e7e..c85cc4952 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,6 +38,10 @@ pipeline {
             steps ( timeout(time: 1, unit: 'HOURS') ) {
                 script {
                     HOME = "${WORKSPACE}/TESTDIR"
+                    pullRequest.addLabel("CI-${machine}-Building")
+                    if (pullRequest.labels.contains("CI-${machine}-Ready")) {
+                        pullRequest.removeLabel("CI-${machine}-Ready")
+                    }
                 }
             }
         }
@@ -75,7 +79,7 @@ pipeline {
                                         } else if (system == "gefs") {
                                             // TODO: need to add gefs build arguments from a yaml file
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
@@ -95,7 +99,7 @@ pipeline {
                 script {
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
-                    //pullRequest.removeLabel("CI-${machine}-Building")
+                    pullRequest.removeLabel("CI-${machine}-Building")
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
@@ -134,15 +138,9 @@ pipeline {
                                 sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             } catch (Exception e) {
-                                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                                    fileContent.eachLine { line ->
-                                        archiveArtifacts artifacts: "${line}", fingerprint: true
-                                    }
+                                pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
+                                error("Failed to run experiments ${Case} on ${machine}")
                                 }
-                            }
-                                    pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
-                                    error("Failed to run experiments ${Case} on ${machine}")
                             }
                         }
                     }
@@ -172,6 +170,12 @@ pipeline {
                 pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                        fileContent.eachLine { line ->
+                        archiveArtifacts artifacts: "${line}", fingerprint: true
+                    }
+                }
             }
         }
     }
-- 
GitLab


From 2e159cd77f92005e2bc08693730e7a4abc347b77 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 17:29:25 +0000
Subject: [PATCH 319/447] added throttle to build

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c85cc4952..64d69c184 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -39,13 +39,14 @@ pipeline {
                 script {
                     HOME = "${WORKSPACE}/TESTDIR"
                     pullRequest.addLabel("CI-${machine}-Building")
-                    if (pullRequest.labels.contains("CI-${machine}-Ready")) {
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
                     }
                 }
             }
         }
 
+        properties([throttle(['Build'])])
         stage('Build') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-- 
GitLab


From e1f054f343c92b8e5518b67bea69f5a09b5f0f34 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 17:33:53 +0000
Subject: [PATCH 320/447] remove prop from throttle

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 64d69c184..07e4f8148 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,7 +46,7 @@ pipeline {
             }
         }
 
-        properties([throttle(['Build'])])
+        throttle(['Build']) {
         stage('Build') {
             matrix {
                 agent { label "${MACHINE}-emc" }
@@ -92,7 +92,7 @@ pipeline {
                     }
                 }
             }
-        }
+        } }
 
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
-- 
GitLab


From 03b551f72979bf806c9e0c3a8f02bc86a88eacf7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 17:41:46 +0000
Subject: [PATCH 321/447] moveed throttle inside options block of stage build

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 07e4f8148..bc231800d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,10 +46,12 @@ pipeline {
             }
         }
 
-        throttle(['Build']) {
         stage('Build') {
             matrix {
                 agent { label "${MACHINE}-emc" }
+                options {
+                    throttle(['Building'])
+                }
                 axes {
                     axis { 
                         name "system"
@@ -92,7 +94,7 @@ pipeline {
                     }
                 }
             }
-        } }
+        }
 
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
-- 
GitLab


From 2329c4a877688d803d6123e373264c21c20f87bf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 17:52:01 +0000
Subject: [PATCH 322/447] updated fixed label check code

---
 Jenkinsfile | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bc231800d..33f4733bf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -101,8 +101,9 @@ pipeline {
             steps {
                 script {
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    //TODO cannot get pullRequest.labels.contains("CI-${machine}-Building") to work
-                    pullRequest.removeLabel("CI-${machine}-Building")
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
+                         pullRequest.removeLabel("CI-${machine}-Building")
+                    }
                     pullRequest.addLabel("CI-${machine}-Running")
                 }
             }
@@ -154,29 +155,29 @@ pipeline {
     }
 
     post {
+        always {
+            if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) {
+                pullRequest.removeLabel("CI-${machine}-Running")
+            }
+        }
         success {
             script {
-                //if ( pullRequest.labels.contains( "CI-${machine}-Running" ) ) {
-                //   pullRequest.removeLabel("CI-${machine}-Running")
-                //}
-                // TODO: contains mehthod does not work
-                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Passed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
             }
-            cleanWs()
         }
         failure {
             script {
-                pullRequest.removeLabel("CI-${machine}-Running")
                 pullRequest.addLabel("CI-${machine}-Failed")
                 def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                 pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
                 if (fileExists('${HOME}/RUNTESTS/ci.log')) {
                     def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
                         fileContent.eachLine { line ->
-                        archiveArtifacts artifacts: "${line}", fingerprint: true
+                        if( line.contains(".log")) {
+                            archiveArtifacts artifacts: "${line}", fingerprint: true
+                        }
                     }
                 }
             }
-- 
GitLab


From fc7a3a6abd901535472c9f45016e1d2228d01fb3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 17:54:53 +0000
Subject: [PATCH 323/447] ndded script block in post always

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 33f4733bf..5eb9572a2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -156,8 +156,10 @@ pipeline {
 
     post {
         always {
-            if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) {
-                pullRequest.removeLabel("CI-${machine}-Running")
+            script {
+               if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) {
+                    pullRequest.removeLabel("CI-${machine}-Running")
+                }
             }
         }
         success {
-- 
GitLab


From 5c4d4a6e089bddbe8f12217b618cc5c06d7854be Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 18:06:28 +0000
Subject: [PATCH 324/447] added confgired catigory to throttle

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5eb9572a2..1d8f273fa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
             matrix {
                 agent { label "${MACHINE}-emc" }
                 options {
-                    throttle(['Building'])
+                    throttle(['Matrix_build'])
                 }
                 axes {
                     axis { 
-- 
GitLab


From c0b386f70ae8f1137e5b38bf20c499b2f7f7383c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 18:29:38 +0000
Subject: [PATCH 325/447] took out throttle option and added limit j to 2 on
 both concurrent builds

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1d8f273fa..d9c5ace51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,9 +49,9 @@ pipeline {
         stage('Build') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['Matrix_build'])
-                }
+                //options {
+                //    throttle(['Matrix_build'])
+                //}
                 axes {
                     axis { 
                         name "system"
@@ -75,14 +75,14 @@ pipeline {
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         if (system == "gfs") {
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu -j 2", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
                                             // TODO: need to add gefs build arguments from a yaml file
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu -j 2", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
-- 
GitLab


From e896a1c5a9238fd887e646dbfb3b204a2f957f36 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 29 Jan 2024 18:51:36 +0000
Subject: [PATCH 326/447] made remove all labels with machine in post alwasy a
 conventional loop

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d9c5ace51..273ac500f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -157,8 +157,10 @@ pipeline {
     post {
         always {
             script {
-               if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Running") } ) {
-                    pullRequest.removeLabel("CI-${machine}-Running")
+                for (label in pullRequest.labels) {
+                    if (label.contains("${machine}")) {
+                        pullRequest.removeLabel(label)
+                    }
                 }
             }
         }
-- 
GitLab


From ec432dafb6514abebd6dbeb2ba19ed6963e36168 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 29 Jan 2024 15:27:35 -0500
Subject: [PATCH 327/447] Update Jenkinsfile

just removed one system and try just building gfs
---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 273ac500f..024072ee8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -55,7 +55,7 @@ pipeline {
                 axes {
                     axis { 
                         name "system"
-                        values "gfs", "gefs"
+                        values "gfs"
                     }
                 }
                 stages {
@@ -75,7 +75,7 @@ pipeline {
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         if (system == "gfs") {
                                             dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu -j 2", returnStatus: false)
+                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
-- 
GitLab


From 189025956df91c0f83d7eb82cd76b8bdf6730353 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:12:32 +0000
Subject: [PATCH 328/447] added to matrix builds to make it sequential and
 added yaml reads

---
 Jenkinsfile | 122 +++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 116 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 024072ee8..b70695e64 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,7 +46,7 @@ pipeline {
             }
         }
 
-        stage('Build') {
+        stage('Build GEFS') {
             matrix {
                 agent { label "${MACHINE}-emc" }
                 //options {
@@ -55,6 +55,8 @@ pipeline {
                 axes {
                     axis { 
                         name "system"
+                        //values "gfs", "gefs"
+                        values "gefs"
                         values "gfs"
                     }
                 }
@@ -73,14 +75,122 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         if (system == "gfs") {
+                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            echo "build args: ${build_args}"
+                                            dir("${HOMEgfs}/sorc") {
+                                                sh( script: "./${build_args}", returnStatus: false)
+                                                sh( script: "./link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            }
+                                        } else if (system == "gefs") {
+                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        stage('Build GFS') {
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                //options {
+                //    throttle(['Matrix_build'])
+                //}
+                axes {
+                    axis { 
+                        name "system"
+                        //values "gfs", "gefs"
+                        values "gfs"
+                    }
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                dir(HOMEgfs) {
+                                    checkout scm
+                                    env.MACHINE_ID = MACHINE
+                                    if (fileExists("sorc/BUILT_semaphor")) {
+                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                    } else {
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        if (system == "gfs") {
+                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            echo "build args: ${build_args}"
+                                            dir("${HOMEgfs}/sorc") {
+                                                sh( script: "./${build_args}", returnStatus: false)
+                                                sh( script: "./link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        stage('Build GFS') {
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                //options {
+                //    throttle(['Matrix_build'])
+                //}
+                axes {
+                    axis { 
+                        name "system"
+                        //values "gfs", "gefs"
+                        values "gfs"
+                    }
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                dir(HOMEgfs) {
+                                    checkout scm
+                                    env.MACHINE_ID = MACHINE
+                                    if (fileExists("sorc/BUILT_semaphor")) {
+                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                    } else {
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        if (system == "gfs") {
+                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            echo "build args: ${build_args}"
+                                            dir("${HOMEgfs}/sorc") {
+                                                sh( script: "./${build_args}", returnStatus: false)
+                                                sh( script: "./link_workflow.sh", returnStatus: false)
+                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            }
                                         } else if (system == "gefs") {
-                                            // TODO: need to add gefs build arguments from a yaml file
+                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./build_all.sh -gu -j 2", returnStatus: false)
                                                 sh( script: "./link_workflow.sh", returnStatus: false)
@@ -96,6 +206,7 @@ pipeline {
             }
         }
 
+
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
             steps {
@@ -123,11 +234,10 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                                 script {
+                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml"
+                                    system = yaml_case.experiment.system
+                                    def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    def HOMEgfs = "${HOME}/gfs"
-                                    env.HOME = HOMEgfs
-                                    sh( script: "rm -Rf ${RUNTESTS}/EXPDIR/${Case}_*" )
-                                    sh( script: "rm -Rf ${RUNTESTS}/COMROOT/${Case}_*" )
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 }
                         }
-- 
GitLab


From f3be7f36df6b00cdba38f269fe23f72f8551766f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:18:13 +0000
Subject: [PATCH 329/447] had extra values line in first matrix build

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b70695e64..ae4e682e7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -57,7 +57,6 @@ pipeline {
                         name "system"
                         //values "gfs", "gefs"
                         values "gefs"
-                        values "gfs"
                     }
                 }
                 stages {
-- 
GitLab


From 8a263936b3bfb7853ff23cf299978e2efa2561b1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:34:28 +0000
Subject: [PATCH 330/447] had an extra pasted GFS build

---
 Jenkinsfile | 57 -----------------------------------------------------
 1 file changed, 57 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ae4e682e7..9e1b75a6d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -149,63 +149,6 @@ pipeline {
             }
         }
 
-
-        stage('Build GFS') {
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['Matrix_build'])
-                //}
-                axes {
-                    axis { 
-                        name "system"
-                        //values "gfs", "gefs"
-                        values "gfs"
-                    }
-                }
-                stages {
-                    stage("build system") {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/${system}"
-                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
-                                    checkout scm
-                                    env.MACHINE_ID = MACHINE
-                                    if (fileExists("sorc/BUILT_semaphor")) {
-                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                                    } else {
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
-                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        if (system == "gfs") {
-                                            def build_args = builds_file.find { it.system == system }.build_args
-                                            echo "build args: ${build_args}"
-                                            dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./${build_args}", returnStatus: false)
-                                                sh( script: "./link_workflow.sh", returnStatus: false)
-                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            }
-                                        } else if (system == "gefs") {
-                                            def build_args = builds_file.find { it.system == system }.build_args
-                                            echo "build args: ${build_args}"
-                                            dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu -j 2", returnStatus: false)
-                                                sh( script: "./link_workflow.sh", returnStatus: false)
-                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
             steps {
-- 
GitLab


From cbf32bc48979786a35a92f92b1952cdbb176b669 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:41:18 +0000
Subject: [PATCH 331/447] added build yaml file in cases yamls

---
 ci/cases/yamls/build.yaml | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 ci/cases/yamls/build.yaml

diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
new file mode 100644
index 000000000..20c80fe29
--- /dev/null
+++ b/ci/cases/yamls/build.yaml
@@ -0,0 +1,3 @@
+builds:
+- gefs: './build_all.sh -gu'
+- gfs: './build_all.sh -gu'
-- 
GitLab


From bed6fa8bf7047fa8b96ab813730154c24c67931c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:51:14 +0000
Subject: [PATCH 332/447] variable key had to be imutable

---
 Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9e1b75a6d..7cd2747de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,7 +76,8 @@ pipeline {
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         if (system == "gfs") {
-                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            def system_key = string( system )
+                                            def build_args = builds_file.find { it.system_key == system_key }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./${build_args}", returnStatus: false)
@@ -84,7 +85,8 @@ pipeline {
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
-                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            def system_key = string( system )
+                                            def build_args = builds_file.find { it.system_key == system_key }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
-- 
GitLab


From e4426fcb4587b6c3ac1bb5773c223f031a5cedb1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 19:55:00 +0000
Subject: [PATCH 333/447] testing imuable key

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7cd2747de..f64b6d4fa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -85,7 +85,8 @@ pipeline {
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
-                                            def system_key = string( system )
+                                            // def system_key = string( system )
+                                            def system_key = "gefs"
                                             def build_args = builds_file.find { it.system_key == system_key }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
-- 
GitLab


From d8bbea93972d439554e5a3eee7ff31574162dc47 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 20:11:43 +0000
Subject: [PATCH 334/447] testing yaml output

---
 Jenkinsfile | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f64b6d4fa..9408cbf9d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -67,17 +67,17 @@ pipeline {
                                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
-                                    checkout scm
+                                    //checkout scm
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        def build_args = builds_file['builds']
+                                        echo "build args: ${build_args}"
                                         if (system == "gfs") {
-                                            def system_key = string( system )
-                                            def build_args = builds_file.find { it.system_key == system_key }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./${build_args}", returnStatus: false)
@@ -85,9 +85,6 @@ pipeline {
                                                 sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                             }
                                         } else if (system == "gefs") {
-                                            // def system_key = string( system )
-                                            def system_key = "gefs"
-                                            def build_args = builds_file.find { it.system_key == system_key }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
-- 
GitLab


From 4a03f8760f0f5be440aae8765c490795301f7f1b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 20:26:39 +0000
Subject: [PATCH 335/447] traverse build args

---
 Jenkinsfile | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9408cbf9d..1fae34f10 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,21 +76,15 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args = builds_file['builds']
+                                        build_args.value.each{ build ->
+                                        if (build.key == system) {
+                                            build_args = build.value
+                                        }
                                         echo "build args: ${build_args}"
-                                        if (system == "gfs") {
-                                            echo "build args: ${build_args}"
-                                            dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./${build_args}", returnStatus: false)
-                                                sh( script: "./link_workflow.sh", returnStatus: false)
-                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            }
-                                        } else if (system == "gefs") {
-                                            echo "build args: ${build_args}"
-                                            dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./build_all.sh -gu -j 4", returnStatus: false)
-                                                sh( script: "./link_workflow.sh", returnStatus: false)
-                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            }
+                                        dir("${HOMEgfs}/sorc") {
+                                            sh( script: "${build_args}", returnStatus: false)
+                                            sh( script: "./link_workflow.sh", returnStatus: false)
+                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
                                 }
-- 
GitLab


From 51ab85f6bc78de24b9ce80adaa7db3cd6a240a5f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 20:31:54 +0000
Subject: [PATCH 336/447] missed } on iter

---
 Jenkinsfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1fae34f10..ff22e226a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -77,12 +77,13 @@ pipeline {
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args = builds_file['builds']
                                         build_args.value.each{ build ->
-                                        if (build.key == system) {
-                                            build_args = build.value
+                                            if (build.key == system) {
+                                               build_args = build.value
+                                            } 
                                         }
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: false)
+                                            sh( script: "", returnStatus: false)
                                             sh( script: "./link_workflow.sh", returnStatus: false)
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
-- 
GitLab


From f2a018a3cf59c56494ca3dfa171503fa27f5643c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 20:42:29 +0000
Subject: [PATCH 337/447] use get in yaml

---
 Jenkinsfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ff22e226a..a8b43f0b2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -75,12 +75,8 @@ pipeline {
                                     } else {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        def build_args = builds_file['builds']
-                                        build_args.value.each{ build ->
-                                            if (build.key == system) {
-                                               build_args = build.value
-                                            } 
-                                        }
+                                        def build_args_list = builds_file['builds']
+                                        def build_args = build_args_list.get(system)
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
-- 
GitLab


From 59f7aeeb1a7ab19204312c2a2d93c090c371c510 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 20:59:39 +0000
Subject: [PATCH 338/447] try it key

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a8b43f0b2..81075c161 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,7 +76,7 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list.get(system)
+                                        def build_args = build_args_list.find { it.key == system }.value
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
-- 
GitLab


From 2910b53dcaee864677be4f4e59c081bd7ecf55e3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:15:49 +0000
Subject: [PATCH 339/447] just system

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 81075c161..4909c2dd2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,7 +76,8 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list.find { it.key == system }.value
+                                        echo "build args list: ${build_args_list}"
+                                        def build_args = build_args_list[system]
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
-- 
GitLab


From cbc9e2282cc3ce66a52f99d82d61e6fd9ec7b602 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:25:32 +0000
Subject: [PATCH 340/447] to string

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4909c2dd2..a1de84130 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -77,7 +77,7 @@ pipeline {
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         echo "build args list: ${build_args_list}"
-                                        def build_args = build_args_list[system]
+                                        def build_args = build_args_list[system].toString()
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
-- 
GitLab


From b3738650de5fb21017ad63c158a73f560d97572e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:32:55 +0000
Subject: [PATCH 341/447] to string

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a1de84130..e6ee571e6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -77,7 +77,7 @@ pipeline {
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         echo "build args list: ${build_args_list}"
-                                        def build_args = build_args_list[system].toString()
+                                        def build_args = build_args_list[system].join()
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
-- 
GitLab


From a0fd397894ba2e019980f701299a51af17a6ed66 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:39:57 +0000
Subject: [PATCH 342/447] still trying

---
 Jenkinsfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e6ee571e6..784ba7608 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -77,8 +77,10 @@ pipeline {
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         echo "build args list: ${build_args_list}"
-                                        def build_args = build_args_list[system].join()
+                                        def build_args = build_args_list[system]
                                         echo "build args: ${build_args}"
+                                        def test = build_args.join(" ")
+                                        echo "test: ${test}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "", returnStatus: false)
                                             sh( script: "./link_workflow.sh", returnStatus: false)
@@ -115,16 +117,16 @@ pipeline {
                                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
-                                    checkout scm
+                                    //checkout scm
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         if (system == "gfs") {
-                                            def build_args = builds_file.find { it.system == system }.build_args
+                                            //def build_args = builds_file.find { it.system == system }.build_args
                                             echo "build args: ${build_args}"
                                             dir("${HOMEgfs}/sorc") {
                                                 sh( script: "./${build_args}", returnStatus: false)
-- 
GitLab


From 09d6eb82d288298e7844cc71730d9c5910a33270 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:46:32 +0000
Subject: [PATCH 343/447] still foo

---
 Jenkinsfile | 47 -----------------------------------------------
 1 file changed, 47 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 784ba7608..fde9072bb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -96,53 +96,6 @@ pipeline {
         }
 
 
-        stage('Build GFS') {
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['Matrix_build'])
-                //}
-                axes {
-                    axis { 
-                        name "system"
-                        //values "gfs", "gefs"
-                        values "gfs"
-                    }
-                }
-                stages {
-                    stage("build system") {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/${system}"
-                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
-                                    //checkout scm
-                                    env.MACHINE_ID = MACHINE
-                                    if (fileExists("sorc/BUILT_semaphor")) {
-                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                                    } else {
-                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
-                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        if (system == "gfs") {
-                                            //def build_args = builds_file.find { it.system == system }.build_args
-                                            echo "build args: ${build_args}"
-                                            dir("${HOMEgfs}/sorc") {
-                                                sh( script: "./${build_args}", returnStatus: false)
-                                                sh( script: "./link_workflow.sh", returnStatus: false)
-                                                sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
             steps {
-- 
GitLab


From 01e09baecf493a4564c2a01c0645ba1451b19b24 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:53:46 +0000
Subject: [PATCH 344/447] finally

---
 Jenkinsfile | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fde9072bb..1b9fcd728 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,13 +76,10 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-                                        echo "build args list: ${build_args_list}"
-                                        def build_args = build_args_list[system]
+                                        def build_args = build_args_list[system].join(" ")
                                         echo "build args: ${build_args}"
-                                        def test = build_args.join(" ")
-                                        echo "test: ${test}"
                                         dir("${HOMEgfs}/sorc") {
-                                            sh( script: "", returnStatus: false)
+                                            sh( script: "${build_args}", returnStatus: false)
                                             sh( script: "./link_workflow.sh", returnStatus: false)
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
-- 
GitLab


From bb7ff253d3d54d46dcd8fcf2fa3a79fb447f930f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 21:56:46 +0000
Subject: [PATCH 345/447] finally then trim

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1b9fcd728..76aad3bbe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,7 +76,7 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(" ")
+                                        def build_args = build_args_list[system].join(" ").trim()
                                         echo "build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "${build_args}", returnStatus: false)
-- 
GitLab


From 01567473201624bf473f35e26052f98c1102d16a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 22:03:07 +0000
Subject: [PATCH 346/447] finally then trim and nulls

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 76aad3bbe..b3a4b6f75 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,8 +76,8 @@ pipeline {
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(" ").trim()
-                                        echo "build args: ${build_args}"
+                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
+                                        echo "trim build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "${build_args}", returnStatus: false)
                                             sh( script: "./link_workflow.sh", returnStatus: false)
-- 
GitLab


From e9a6b70dbebd0796acd2b0403a8e5e955f0814d4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 30 Jan 2024 22:06:11 +0000
Subject: [PATCH 347/447] got args to work

---
 Jenkinsfile | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index b3a4b6f75..2123f8c90 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,6 +46,52 @@ pipeline {
             }
         }
 
+        stage('Build GFS') {
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                //options {
+                //    throttle(['Matrix_build'])
+                //}
+                axes {
+                    axis { 
+                        name "system"
+                        //values "gfs", "gefs"
+                        values "gfs"
+                    }
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                dir(HOMEgfs) {
+                                    //checkout scm
+                                    env.MACHINE_ID = MACHINE
+                                    if (fileExists("sorc/BUILT_semaphor")) {
+                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                    } else {
+                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        def build_args_list = builds_file['builds']
+                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
+                                        echo "trim build args: ${build_args}"
+                                        dir("${HOMEgfs}/sorc") {
+                                            sh( script: "${build_args}", returnStatus: false)
+                                            sh( script: "./link_workflow.sh", returnStatus: false)
+                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         stage('Build GEFS') {
             matrix {
                 agent { label "${MACHINE}-emc" }
@@ -93,6 +139,7 @@ pipeline {
         }
 
 
+
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
             steps {
-- 
GitLab


From c4dcbe68b89244987f41fdc5a5d276b996bff2c3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 01:01:56 +0000
Subject: [PATCH 348/447] put clone back in

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2123f8c90..2835c719c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -113,13 +113,13 @@ pipeline {
                                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
-                                    //checkout scm
+                                    checkout scm
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From ec6ba94c290d4c58b91a8e510dbb42da228812b2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 15:51:59 +0000
Subject: [PATCH 349/447] added back throttled build and removed second build
 block

---
 Jenkinsfile | 59 +++++------------------------------------------------
 1 file changed, 5 insertions(+), 54 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2835c719c..85f7373ce 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -46,63 +46,17 @@ pipeline {
             }
         }
 
-        stage('Build GFS') {
+        stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['Matrix_build'])
-                //}
-                axes {
-                    axis { 
-                        name "system"
-                        //values "gfs", "gefs"
-                        values "gfs"
-                    }
-                }
-                stages {
-                    stage("build system") {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/${system}"
-                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
-                                    //checkout scm
-                                    env.MACHINE_ID = MACHINE
-                                    if (fileExists("sorc/BUILT_semaphor")) {
-                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                                    } else {
-                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
-                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-                                        echo "trim build args: ${build_args}"
-                                        dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: false)
-                                            sh( script: "./link_workflow.sh", returnStatus: false)
-                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
+                options {
+                    throttle(['global_matrix_build'])
                 }
-            }
-        }
-
-        stage('Build GEFS') {
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['Matrix_build'])
-                //}
                 axes {
                     axis { 
                         name "system"
-                        //values "gfs", "gefs"
-                        values "gefs"
+                        values "gfs", "gefs"
+                        values "gfs"
                     }
                 }
                 stages {
@@ -123,7 +77,6 @@ pipeline {
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-                                        echo "trim build args: ${build_args}"
                                         dir("${HOMEgfs}/sorc") {
                                             sh( script: "${build_args}", returnStatus: false)
                                             sh( script: "./link_workflow.sh", returnStatus: false)
@@ -138,8 +91,6 @@ pipeline {
             }
         }
 
-
-
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
             steps {
-- 
GitLab


From 85522b7ee985d4139ba9583068662cf9ac2486b2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 16:03:42 +0000
Subject: [PATCH 350/447] removed extra leftover values line in build matrix

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 85f7373ce..bcecca037 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -56,7 +56,6 @@ pipeline {
                     axis { 
                         name "system"
                         values "gfs", "gefs"
-                        values "gfs"
                     }
                 }
                 stages {
-- 
GitLab


From 1c77e99b0f0bc0a320dd950ffd72862963a67afa Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 18:32:24 +0000
Subject: [PATCH 351/447] removed non yaml lines from cases so snakeyaml could
 read it

---
 Jenkinsfile                  | 3 ++-
 ci/scripts/utils/ci_utils.sh | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bcecca037..b861369c2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -117,7 +117,8 @@ pipeline {
                     stage('Create Experiment') {
                         steps {
                                 script {
-                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml"
+                                    sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
+                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 6b9312257..340288a38 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -97,6 +97,8 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  "${HOME}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
+
+  "${HOMEgfs}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}"
 
 }
-- 
GitLab


From 773164aa12741e94bf91dc5c1eb8454ad187865f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 19:53:15 +0000
Subject: [PATCH 352/447] added override in create experment and removed stubed
 scripts

---
 ci/scripts/run-check_ci_stub.sh | 11 -----------
 ci/scripts/utils/ci_utils.sh    |  2 +-
 sorc/build_all_stub.sh          |  5 -----
 3 files changed, 1 insertion(+), 17 deletions(-)
 delete mode 100755 ci/scripts/run-check_ci_stub.sh
 delete mode 100755 sorc/build_all_stub.sh

diff --git a/ci/scripts/run-check_ci_stub.sh b/ci/scripts/run-check_ci_stub.sh
deleted file mode 100755
index 23d3da837..000000000
--- a/ci/scripts/run-check_ci_stub.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-HOMEgfs=$1
-pslot=$2
-
-echo -e "HOMEgfs: ${HOMEgfs}\npslpot: ${pslot}\n in ${PWD}" >> "${HOMEgfs}/ci/scripts/run-check_ci_stub.log"
-if [[ ${pslot} =~ "C48_S2SW" ]]; then
-     exit 1
-fi
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 340288a38..7422d3457 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -99,6 +99,6 @@ function create_experiment () {
 
   # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
 
-  "${HOMEgfs}/${system}/workflow/create_experiment.py" --yaml "${yaml_config}"
+  "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
 
 }
diff --git a/sorc/build_all_stub.sh b/sorc/build_all_stub.sh
deleted file mode 100755
index 6221b6bb5..000000000
--- a/sorc/build_all_stub.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-echo "Ran build all stub in ${PWD}" > "BUILD_STUB_RAN_$(date +%d%H%M)"
-- 
GitLab


From 9747d7affec401d554260de48ac99edbd0cf3c19 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 31 Jan 2024 20:33:10 +0000
Subject: [PATCH 353/447] moved scm after semephore test

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b861369c2..0bb9f747c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,12 +66,12 @@ pipeline {
                                 properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
-                                    checkout scm
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
+                                        checkout scm
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-- 
GitLab


From 6d439b66068b539d4d9d2e4f511a4393dbf9ddc0 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 31 Jan 2024 18:01:25 -0500
Subject: [PATCH 354/447] Update Jenkinsfile

try a build without a manual git submodule recursive and see if scm can do it correctly
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0bb9f747c..d829b1e62 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,7 +72,7 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         checkout scm
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From a7ff014d423ef45767f0d37edad9720085435440 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 31 Jan 2024 18:04:40 -0500
Subject: [PATCH 355/447] Update Jenkinsfile

no for separate PR test on Orion
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d829b1e62..f9b7ee322 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,7 +1,7 @@
 def MACHINE = 'none'
 def machine = 'none'
 def HOME = 'none'
-
+// NO OP for Orion seperate PR
 pipeline {
     agent { label 'built-in' }
 
-- 
GitLab


From c2e0f82c8b2191edb772c6444ef16b9bbe0a0a4e Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 31 Jan 2024 21:09:47 -0500
Subject: [PATCH 356/447] Update Jenkinsfile

put back in submodle upate
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f9b7ee322..09a2c128f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,7 +72,7 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         checkout scm
-                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From da9f7cd11a52a3e2b382802307a60e50a90eed67 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:05:51 +0000
Subject: [PATCH 357/447] added pre checkout in TESTDIR added source of gwsetup
 and changed first dir to ws

---
 Jenkinsfile | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 09a2c128f..5fc45fa06 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,7 +1,15 @@
 def MACHINE = 'none'
 def machine = 'none'
 def HOME = 'none'
-// NO OP for Orion seperate PR
+
+checkout([
+    $class: 'GitSCM', 
+    branches: scm.branches, 
+    doGenerateSubmoduleConfigurations: false, 
+    extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], 
+    userRemoteConfigs: scm.userRemoteConfigs
+])
+
 pipeline {
     agent { label 'built-in' }
 
@@ -37,7 +45,10 @@ pipeline {
             agent { label "${MACHINE}-emc" }
             steps ( timeout(time: 1, unit: 'HOURS') ) {
                 script {
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
+                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
+                    checkout
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -63,16 +74,16 @@ pipeline {
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-                                properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
+                                ws(HOMEgfs) {
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
+                                        sh( script: "source ${HOME/workflow/gw_setup.sh}", returnStatus: true)
                                         checkout scm
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From 3662f07d878fffc03d227ee9a9c245d1953b0d49 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:12:06 +0000
Subject: [PATCH 358/447] added git to hera gw setup module load

---
 modulefiles/module_gwsetup.hera.lua | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua
index c86cac7b0..ab7223372 100644
--- a/modulefiles/module_gwsetup.hera.lua
+++ b/modulefiles/module_gwsetup.hera.lua
@@ -3,6 +3,7 @@ Load environment to run GFS workflow setup scripts on Hera
 ]])
 
 load(pathJoin("rocoto"))
+load(pathJoin("git","2.40.0"))
 
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
-- 
GitLab


From 5c2eb59b1e691bf05cc4dfff3ad4717fb6c50a17 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:20:02 +0000
Subject: [PATCH 359/447] made shallow checkout a function call

---
 Jenkinsfile | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5fc45fa06..dc82e2b4f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,13 +2,15 @@ def MACHINE = 'none'
 def machine = 'none'
 def HOME = 'none'
 
-checkout([
-    $class: 'GitSCM', 
-    branches: scm.branches, 
-    doGenerateSubmoduleConfigurations: false, 
-    extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], 
-    userRemoteConfigs: scm.userRemoteConfigs
-])
+def checkout_shallow () {
+    checkout([
+       $class: 'GitSCM', 
+       branches: scm.branches, 
+       doGenerateSubmoduleConfigurations: false, 
+       extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], 
+       userRemoteConfigs: scm.userRemoteConfigs
+    ])
+}
 
 pipeline {
     agent { label 'built-in' }
@@ -48,7 +50,7 @@ pipeline {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
                     sh( script: "mkdir -p ${HOME}", returnStatus: true)
-                    checkout
+                    checkout_shallow()
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
-- 
GitLab


From be7b72e5747efedd0aa06d0b9db8b277e2c76188 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:23:09 +0000
Subject: [PATCH 360/447] misplaced bracket in source of gw_setup

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dc82e2b4f..5eaf25970 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -83,7 +83,7 @@ pipeline {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        sh( script: "source ${HOME/workflow/gw_setup.sh}", returnStatus: true)
+                                        sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true)
                                         checkout scm
                                         //sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-- 
GitLab


From aeaa5cfba670bdc5f5739cd7427834c54b089b8f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:34:35 +0000
Subject: [PATCH 361/447] cd into TESTDIR for shallow clone

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5eaf25970..856245acc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,9 @@ pipeline {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
                     sh( script: "mkdir -p ${HOME}", returnStatus: true)
-                    checkout_shallow()
+                    dir("${HOME}") {
+                       checkout_shallow()
+                    }
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
-- 
GitLab


From 8fa786824e021709bcfe4e9e93635e555426e6c6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:42:47 +0000
Subject: [PATCH 362/447] moved git load on gwset hera because it is in the
 stack spac

---
 modulefiles/module_gwsetup.hera.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua
index ab7223372..244a2419e 100644
--- a/modulefiles/module_gwsetup.hera.lua
+++ b/modulefiles/module_gwsetup.hera.lua
@@ -3,7 +3,6 @@ Load environment to run GFS workflow setup scripts on Hera
 ]])
 
 load(pathJoin("rocoto"))
-load(pathJoin("git","2.40.0"))
 
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
@@ -15,5 +14,6 @@ load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
+load(pathJoin("git","2.40.0"))
 
 whatis("Description: GFS run setup environment")
-- 
GitLab


From 76f13ed3b9b058c00a7e96bd8fcfd08fe9c1fcab Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 15:57:20 +0000
Subject: [PATCH 363/447] use shallow checkout and git submoudle recussive
 update directly

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 856245acc..e7b1f5317 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,8 +86,8 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true)
-                                        checkout scm
-                                        //sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        checkout_shallow()
+                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From 2685a85cf7f0428c1b9750543642aaa4e53c43c7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 17:49:03 +0000
Subject: [PATCH 364/447] configured scm shallow and using git modules
 recursive from host

---
 Jenkinsfile | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e7b1f5317..324d166e7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,16 +2,6 @@ def MACHINE = 'none'
 def machine = 'none'
 def HOME = 'none'
 
-def checkout_shallow () {
-    checkout([
-       $class: 'GitSCM', 
-       branches: scm.branches, 
-       doGenerateSubmoduleConfigurations: false, 
-       extensions: [[$class: 'SubmoduleOption', disableSubmodules: true]], 
-       userRemoteConfigs: scm.userRemoteConfigs
-    ])
-}
-
 pipeline {
     agent { label 'built-in' }
 
@@ -86,7 +76,7 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true)
-                                        checkout_shallow()
+                                        checkout scm
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-- 
GitLab


From 0944fd418dea5e94b758bb3cbe8ea7d5f6b6feeb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 17:53:13 +0000
Subject: [PATCH 365/447] still had shallow call and removed it

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 324d166e7..1ee23a771 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
                     HOME = "${WORKSPACE}/TESTDIR"
                     sh( script: "mkdir -p ${HOME}", returnStatus: true)
                     dir("${HOME}") {
-                       checkout_shallow()
+                       checkout scm
                     }
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
-- 
GitLab


From a1ba03b4d037caddaa246e003334be8ea041b6c2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 18:18:48 +0000
Subject: [PATCH 366/447] put dir back at attempt to use ws

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1ee23a771..cd7d890a6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -69,7 +69,7 @@ pipeline {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                ws(HOMEgfs) {
+                                dir(HOMEgfs) {
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("sorc/BUILT_semaphor")) {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-- 
GitLab


From de514d90c2f7d2d59dbb663f3f6aa0855c87fc3f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 1 Feb 2024 19:36:18 +0000
Subject: [PATCH 367/447] take out pre checkout and just use submodule update
 directly

---
 Jenkinsfile | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index cd7d890a6..4ef3cf167 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,9 +40,6 @@ pipeline {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
                     sh( script: "mkdir -p ${HOME}", returnStatus: true)
-                    dir("${HOME}") {
-                       checkout scm
-                    }
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -75,7 +72,6 @@ pipeline {
                                         sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        sh( script: "source ${HOME}/workflow/gw_setup.sh", returnStatus: true)
                                         checkout scm
                                         sh( script: "git submodule update --init --recursive", returnStatus: true) 
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-- 
GitLab


From a80666e0815b21e2bc31d94d34933ea2b955f598 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 16:23:37 +0000
Subject: [PATCH 368/447] stub build all to just gdas

---
 Jenkinsfile               | 2 +-
 ci/cases/yamls/build.yaml | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4ef3cf167..fdb23aadf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -73,7 +73,7 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         checkout scm
-                                        sh( script: "git submodule update --init --recursive", returnStatus: true) 
+                                        sh( script: "source workflow/gw_setup.sh;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 20c80fe29..663409ee2 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,5 @@
 builds:
-- gefs: './build_all.sh -gu'
-- gfs: './build_all.sh -gu'
+#- gefs: './build_all.sh -gu'
+#- gfs: './build_all.sh -gu'
+- gefs: './build_gdas.sh -j 4'
+- gfs: './build_gdas.sh -j 8'
-- 
GitLab


From 2e43230d29ca16397c3d80624129f1ba206f38df Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 16:24:49 +0000
Subject: [PATCH 369/447] added git which and version for debugging

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fdb23aadf..435431226 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -73,7 +73,7 @@ pipeline {
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
                                         checkout scm
-                                        sh( script: "source workflow/gw_setup.sh;git submodule update --init --recursive", returnStatus: true)
+                                        sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-- 
GitLab


From 5071db10aee01ecb00aa639382cee76e77b3cbbc Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 16:43:49 +0000
Subject: [PATCH 370/447] put buile lines back in build yaml

---
 ci/cases/yamls/build.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 663409ee2..20c80fe29 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,5 +1,3 @@
 builds:
-#- gefs: './build_all.sh -gu'
-#- gfs: './build_all.sh -gu'
-- gefs: './build_gdas.sh -j 4'
-- gfs: './build_gdas.sh -j 8'
+- gefs: './build_all.sh -gu'
+- gfs: './build_all.sh -gu'
-- 
GitLab


From 948992c3e6f50414aa8717649e226c423b9b6184 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 17:24:46 +0000
Subject: [PATCH 371/447] added a second Jenkins file to test dynamic parallel
 build stages

---
 Jenkinsfile2 | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 186 insertions(+)
 create mode 100644 Jenkinsfile2

diff --git a/Jenkinsfile2 b/Jenkinsfile2
new file mode 100644
index 000000000..435431226
--- /dev/null
+++ b/Jenkinsfile2
@@ -0,0 +1,186 @@
+def MACHINE = 'none'
+def machine = 'none'
+def HOME = 'none'
+
+pipeline {
+    agent { label 'built-in' }
+
+    options {
+        disableConcurrentBuilds(abortPrevious: true)
+        skipDefaultCheckout(true)
+        buildDiscarder(logRotator(numToKeepStr: '3'))
+    }
+
+    stages {
+
+        stage('Get Machine') {
+            agent { label 'built-in' }
+            steps {
+                script {
+                    MACHINE = 'none'
+                    for (label in pullRequest.labels) {
+                        echo "Label: ${label}"
+                        if ((label.matches("CI-Hera-Ready"))) {
+                            MACHINE = 'hera'
+                        } else if ((label.matches("CI-Orion-Ready"))) {
+                            MACHINE = 'orion'
+                        } else if ((label.matches("CI-Hercules-Ready"))) {
+                            MACHINE = 'hercules'
+                        }
+                    }
+                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                }
+            }
+        }
+
+        stage('Get Common Workspace') {
+            agent { label "${MACHINE}-emc" }
+            steps ( timeout(time: 1, unit: 'HOURS') ) {
+                script {
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                    HOME = "${WORKSPACE}/TESTDIR"
+                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
+                    pullRequest.addLabel("CI-${machine}-Building")
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
+                        pullRequest.removeLabel("CI-${machine}-Ready")
+                    }
+                }
+            }
+        }
+
+        stage('Build System') {
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                options {
+                    throttle(['global_matrix_build'])
+                }
+                axes {
+                    axis { 
+                        name "system"
+                        values "gfs", "gefs"
+                    }
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                dir(HOMEgfs) {
+                                    env.MACHINE_ID = MACHINE
+                                    if (fileExists("sorc/BUILT_semaphor")) {
+                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                    } else {
+                                        checkout scm
+                                        sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        def build_args_list = builds_file['builds']
+                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
+                                        dir("${HOMEgfs}/sorc") {
+                                            sh( script: "${build_args}", returnStatus: false)
+                                            sh( script: "./link_workflow.sh", returnStatus: false)
+                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Setup RUNTESTS') {
+            agent { label "${MACHINE}-emc" }
+            steps {
+                script {
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
+                         pullRequest.removeLabel("CI-${machine}-Building")
+                    }
+                    pullRequest.addLabel("CI-${machine}-Running")
+                }
+            }
+
+        }
+
+        stage('Run Tests') {
+            matrix {
+                agent { label "${MACHINE}-emc" }
+                axes {
+                    axis {
+                        name "Case"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                    }
+                }
+                stages {
+                    stage('Create Experiment') {
+                        steps {
+                                script {
+                                    sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
+                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
+                                    system = yaml_case.experiment.system
+                                    def HOMEgfs = "${HOME}/${system}"
+                                    env.RUNTESTS = "${HOME}/RUNTESTS"
+                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                }
+                        }
+                    }
+                    stage('Run Experiments') {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/gfs"
+                                pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                            try {
+                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                            } catch (Exception e) {
+                                pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
+                                error("Failed to run experiments ${Case} on ${machine}")
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    }
+
+    post {
+        always {
+            script {
+                for (label in pullRequest.labels) {
+                    if (label.contains("${machine}")) {
+                        pullRequest.removeLabel(label)
+                    }
+                }
+            }
+        }
+        success {
+            script {
+                pullRequest.addLabel("CI-${machine}-Passed")
+                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
+            }
+        }
+        failure {
+            script {
+                pullRequest.addLabel("CI-${machine}-Failed")
+                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                        fileContent.eachLine { line ->
+                        if( line.contains(".log")) {
+                            archiveArtifacts artifacts: "${line}", fingerprint: true
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}
-- 
GitLab


From e7790ee91ce18e1ce94258c2a60a7bd251fecc7a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 17:31:29 +0000
Subject: [PATCH 372/447] removed Jenkinsfile2 to put it into its own repo

---
 Jenkinsfile2 | 186 ---------------------------------------------------
 1 file changed, 186 deletions(-)
 delete mode 100644 Jenkinsfile2

diff --git a/Jenkinsfile2 b/Jenkinsfile2
deleted file mode 100644
index 435431226..000000000
--- a/Jenkinsfile2
+++ /dev/null
@@ -1,186 +0,0 @@
-def MACHINE = 'none'
-def machine = 'none'
-def HOME = 'none'
-
-pipeline {
-    agent { label 'built-in' }
-
-    options {
-        disableConcurrentBuilds(abortPrevious: true)
-        skipDefaultCheckout(true)
-        buildDiscarder(logRotator(numToKeepStr: '3'))
-    }
-
-    stages {
-
-        stage('Get Machine') {
-            agent { label 'built-in' }
-            steps {
-                script {
-                    MACHINE = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches("CI-Hera-Ready"))) {
-                            MACHINE = 'hera'
-                        } else if ((label.matches("CI-Orion-Ready"))) {
-                            MACHINE = 'orion'
-                        } else if ((label.matches("CI-Hercules-Ready"))) {
-                            MACHINE = 'hercules'
-                        }
-                    }
-                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
-                }
-            }
-        }
-
-        stage('Get Common Workspace') {
-            agent { label "${MACHINE}-emc" }
-            steps ( timeout(time: 1, unit: 'HOURS') ) {
-                script {
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    HOME = "${WORKSPACE}/TESTDIR"
-                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
-                    pullRequest.addLabel("CI-${machine}-Building")
-                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
-                        pullRequest.removeLabel("CI-${machine}-Ready")
-                    }
-                }
-            }
-        }
-
-        stage('Build System') {
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['global_matrix_build'])
-                }
-                axes {
-                    axis { 
-                        name "system"
-                        values "gfs", "gefs"
-                    }
-                }
-                stages {
-                    stage("build system") {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/${system}"
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
-                                    env.MACHINE_ID = MACHINE
-                                    if (fileExists("sorc/BUILT_semaphor")) {
-                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
-                                    } else {
-                                        checkout scm
-                                        sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
-                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-                                        dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: false)
-                                            sh( script: "./link_workflow.sh", returnStatus: false)
-                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        stage('Setup RUNTESTS') {
-            agent { label "${MACHINE}-emc" }
-            steps {
-                script {
-                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
-                         pullRequest.removeLabel("CI-${machine}-Building")
-                    }
-                    pullRequest.addLabel("CI-${machine}-Running")
-                }
-            }
-
-        }
-
-        stage('Run Tests') {
-            matrix {
-                agent { label "${MACHINE}-emc" }
-                axes {
-                    axis {
-                        name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                    }
-                }
-                stages {
-                    stage('Create Experiment') {
-                        steps {
-                                script {
-                                    sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
-                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
-                                    system = yaml_case.experiment.system
-                                    def HOMEgfs = "${HOME}/${system}"
-                                    env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                }
-                        }
-                    }
-                    stage('Run Experiments') {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/gfs"
-                                pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                            try {
-                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
-                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
-                            } catch (Exception e) {
-                                pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
-                                error("Failed to run experiments ${Case} on ${machine}")
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-    }
-
-    post {
-        always {
-            script {
-                for (label in pullRequest.labels) {
-                    if (label.contains("${machine}")) {
-                        pullRequest.removeLabel(label)
-                    }
-                }
-            }
-        }
-        success {
-            script {
-                pullRequest.addLabel("CI-${machine}-Passed")
-                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
-            }
-        }
-        failure {
-            script {
-                pullRequest.addLabel("CI-${machine}-Failed")
-                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                        fileContent.eachLine { line ->
-                        if( line.contains(".log")) {
-                            archiveArtifacts artifacts: "${line}", fingerprint: true
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}
-- 
GitLab


From 29f61f16613f468efc3063fe211b9f9af60a459f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 18:55:07 +0000
Subject: [PATCH 373/447] adding deleteDir to get scm to checkout again

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 435431226..5a33c0e51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,10 +68,11 @@ pipeline {
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
                                     env.MACHINE_ID = MACHINE
-                                    if (fileExists("sorc/BUILT_semaphor")) {
-                                        sh( script: "cat sorc/BUILT_semaphor", returnStdout: true).trim()
+                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
+                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
+                                        deleteDir()
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-- 
GitLab


From 74ff69980a275b5f5a0f54415c07a70dc133d80b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 18:57:01 +0000
Subject: [PATCH 374/447] adding build stubs

---
 ci/cases/yamls/build.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 20c80fe29..6d395d80b 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,5 @@
 builds:
-- gefs: './build_all.sh -gu'
-- gfs: './build_all.sh -gu'
+#- gefs: './build_all.sh -gu'
+#- gfs: './build_all.sh -gu'
+ - gefs: 'echo "Skip gefs build'
+ - gfs: 'echo "Skip gfs build'
-- 
GitLab


From 2978c6caf630b6a2f4ca04de2ed7c3439ffaaa08 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 19:05:34 +0000
Subject: [PATCH 375/447] adding build stubs again

---
 ci/cases/yamls/build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 6d395d80b..357f92f91 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,5 +1,5 @@
 builds:
 #- gefs: './build_all.sh -gu'
 #- gfs: './build_all.sh -gu'
- - gefs: 'echo "Skip gefs build'
- - gfs: 'echo "Skip gfs build'
+ - gefs: 'echo "Skip gefs build"'
+ - gfs: 'echo "Skip gfs build"'
-- 
GitLab


From af48ce8f251d1caf97c235cd808653477498f2ae Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 19:38:24 +0000
Subject: [PATCH 376/447] forgot to add build.yaml updates

---
 Jenkinsfile               | 6 +++---
 ci/cases/yamls/build.yaml | 6 ++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5a33c0e51..be3cbe00b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -51,9 +51,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['global_matrix_build'])
-                }
+                //options {
+                //    throttle(['global_matrix_build'])
+                //}
                 axes {
                     axis { 
                         name "system"
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 357f92f91..56fce131d 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,5 +1,3 @@
 builds:
-#- gefs: './build_all.sh -gu'
-#- gfs: './build_all.sh -gu'
- - gefs: 'echo "Skip gefs build"'
- - gfs: 'echo "Skip gfs build"'
+ - gefs: './build_all.sh -gu'
+ - gfs: './build_all.sh -gu'
\ No newline at end of file
-- 
GitLab


From 348c758c58cfd0fb2911eec2d551144bd58e0b86 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 2 Feb 2024 21:29:56 +0000
Subject: [PATCH 377/447] put throttle back and updated build args for gefs

---
 Jenkinsfile               | 6 +++---
 ci/cases/yamls/build.yaml | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index be3cbe00b..5a33c0e51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -51,9 +51,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['global_matrix_build'])
-                //}
+                options {
+                    throttle(['global_matrix_build'])
+                }
                 axes {
                     axis { 
                         name "system"
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 56fce131d..5398fa188 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,3 @@
 builds:
- - gefs: './build_all.sh -gu'
+ - gefs: './build_all.sh'
  - gfs: './build_all.sh -gu'
\ No newline at end of file
-- 
GitLab


From a0386edaf46795f2e7234c60d1e689bb791e888d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 3 Feb 2024 03:18:01 +0000
Subject: [PATCH 378/447] use rsync to get clones before submodules

---
 Jenkinsfile | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5a33c0e51..01d5be9ba 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -39,7 +39,13 @@ pipeline {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
-                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
+                    sh( script: "mkdir -p ${HOME}/repo_pr", returnStatus: true)
+                    dir("${HOME}/repo_pr") {
+                        checkout scm
+                    }
+                    // TODO get system build dirs from  ${HOME}/repo_pr/ci/cases/yamls/build.yaml
+                    sh( script: "rsync -p ${HOME}/repo_pr  ${HOME}/gfs", returnStatus: true)
+                    sh( script: "rsync -p ${HOME}/repo_pr  ${HOME}/gefs", returnStatus: true)
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -72,8 +78,6 @@ pipeline {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        deleteDir()
-                                        checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-- 
GitLab


From 417688c85474c448390395b119ecc7fd1d4eaaac Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 3 Feb 2024 03:36:09 +0000
Subject: [PATCH 379/447] update rsync line

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 01d5be9ba..0f41524cb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,8 +44,8 @@ pipeline {
                         checkout scm
                     }
                     // TODO get system build dirs from  ${HOME}/repo_pr/ci/cases/yamls/build.yaml
-                    sh( script: "rsync -p ${HOME}/repo_pr  ${HOME}/gfs", returnStatus: true)
-                    sh( script: "rsync -p ${HOME}/repo_pr  ${HOME}/gefs", returnStatus: true)
+                    sh( script: "rsync -a ${HOME}/repo_pr/  ${HOME}/gfs", returnStatus: true)
+                    sh( script: "rsync -a ${HOME}/repo_pr/  ${HOME}/gefs", returnStatus: true)
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -71,7 +71,7 @@ pipeline {
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                // sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-- 
GitLab


From 240a2622b36f8b0ed81411150b0b2fd346f9d624 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 3 Feb 2024 12:43:54 +0000
Subject: [PATCH 380/447] revert to scm and comment out disable current builds

---
 Jenkinsfile | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0f41524cb..e33dbdc4d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -6,7 +6,7 @@ pipeline {
     agent { label 'built-in' }
 
     options {
-        disableConcurrentBuilds(abortPrevious: true)
+        //t disableConcurrentBuilds(abortPrevious: false)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '3'))
     }
@@ -39,13 +39,7 @@ pipeline {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
-                    sh( script: "mkdir -p ${HOME}/repo_pr", returnStatus: true)
-                    dir("${HOME}/repo_pr") {
-                        checkout scm
-                    }
-                    // TODO get system build dirs from  ${HOME}/repo_pr/ci/cases/yamls/build.yaml
-                    sh( script: "rsync -a ${HOME}/repo_pr/  ${HOME}/gfs", returnStatus: true)
-                    sh( script: "rsync -a ${HOME}/repo_pr/  ${HOME}/gefs", returnStatus: true)
+                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -71,13 +65,15 @@ pipeline {
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-                                // sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 dir(HOMEgfs) {
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
+                                        deleteDir()
+                                        checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
-- 
GitLab


From 379958448f7a24e0e4d149ee5c93f2723a0fc96a Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Sun, 4 Feb 2024 14:45:05 -0500
Subject: [PATCH 381/447] Update Jenkinsfile

edit
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e33dbdc4d..e4b6baded 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -6,7 +6,7 @@ pipeline {
     agent { label 'built-in' }
 
     options {
-        //t disableConcurrentBuilds(abortPrevious: false)
+        // disableConcurrentBuilds(abortPrevious: false)
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '3'))
     }
-- 
GitLab


From afc63424b263a455e534d7f5f6f2223a87bab076 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sun, 4 Feb 2024 21:45:14 +0000
Subject: [PATCH 382/447] added checkoutsubdir

---
 Jenkinsfile | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e33dbdc4d..54a359a1b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -6,8 +6,7 @@ pipeline {
     agent { label 'built-in' }
 
     options {
-        //t disableConcurrentBuilds(abortPrevious: false)
-        skipDefaultCheckout(true)
+        skipDefaultCheckout()
         buildDiscarder(logRotator(numToKeepStr: '3'))
     }
 
@@ -51,9 +50,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['global_matrix_build'])
-                }
+                //options {
+                //    throttle(['global_matrix_build'])
+                //}
                 axes {
                     axis { 
                         name "system"
@@ -61,6 +60,9 @@ pipeline {
                     }
                 }
                 stages {
+                    options {
+                       checkoutToSubdirectory('${HOME}/${system}') }
+                    }
                     stage("build system") {
                         steps {
                             script {
@@ -72,7 +74,6 @@ pipeline {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
                                     } else {
-                                        deleteDir()
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-- 
GitLab


From 511ee952d0a26e843817bebec2e3c5c5f96148a4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sun, 4 Feb 2024 21:48:26 +0000
Subject: [PATCH 383/447] extra bracket

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 54a359a1b..2b84fa3c6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -61,7 +61,7 @@ pipeline {
                 }
                 stages {
                     options {
-                       checkoutToSubdirectory('${HOME}/${system}') }
+                       checkoutToSubdirectory('${HOME}/${system}')
                     }
                     stage("build system") {
                         steps {
-- 
GitLab


From 3b3ebc038e0e89f1fa741fddb3391f00764b86cf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sun, 4 Feb 2024 21:51:03 +0000
Subject: [PATCH 384/447] moved options

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2b84fa3c6..b16edaeed 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -60,10 +60,10 @@ pipeline {
                     }
                 }
                 stages {
-                    options {
-                       checkoutToSubdirectory('${HOME}/${system}')
-                    }
                     stage("build system") {
+                        options {
+                          checkoutToSubdirectory('${HOME}/${system}')
+                        }
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-- 
GitLab


From 600cd19024227d7eb5996c782fc66b175620197b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sun, 4 Feb 2024 22:37:32 +0000
Subject: [PATCH 385/447] add no scm in exp

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index b16edaeed..90b07b98e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -95,6 +95,7 @@ pipeline {
 
         stage('Setup RUNTESTS') {
             agent { label "${MACHINE}-emc" }
+            options { skipDefaultCheckout() }
             steps {
                 script {
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-- 
GitLab


From 297a842a633ed8680ca2400f519e5df1cdae8849 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 00:50:48 +0000
Subject: [PATCH 386/447] devult no scm others

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 90b07b98e..4a2b92765 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -111,6 +111,7 @@ pipeline {
         stage('Run Tests') {
             matrix {
                 agent { label "${MACHINE}-emc" }
+                options { skipDefaultCheckout() }
                 axes {
                     axis {
                         name "Case"
@@ -126,7 +127,8 @@ pipeline {
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                    sh( script: "echo 'stub create experiment'", returnStatus: true)
+                                    //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 }
                         }
                     }
-- 
GitLab


From a4174f5b1e8217f540d753a6a097eb2ad72b786d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 01:53:18 +0000
Subject: [PATCH 387/447] put throuttle back in

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4a2b92765..c5f300f6f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,9 +50,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['global_matrix_build'])
-                //}
+                options {
+                    throttle(['global_matrix_build'])
+                }
                 axes {
                     axis { 
                         name "system"
-- 
GitLab


From 74dbb16b25f682d048d330d631d86677c29eb8fa Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 14:29:45 +0000
Subject: [PATCH 388/447] try chaning workspace to get scm to checkout on
 second parallel matrix build

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c5f300f6f..26a913f9c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,7 +7,6 @@ pipeline {
 
     options {
         skipDefaultCheckout()
-        buildDiscarder(logRotator(numToKeepStr: '3'))
     }
 
     stages {
@@ -68,7 +67,8 @@ pipeline {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                dir(HOMEgfs) {
+                                ws(HOMEgfs) {
+                                    checkoutToSubdirectory('${HOME}/${system}')
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-- 
GitLab


From 881697f83fe9dedcffa39b2acf75409ed9e1d8fd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 14:32:37 +0000
Subject: [PATCH 389/447] added stubs for build all to test the Jenkins scm
 issue

---
 ci/cases/yamls/build.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 5398fa188..005db938f 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,5 @@
 builds:
- - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
\ No newline at end of file
+ #- gefs: './build_all.sh'
+ #- gfs: './build_all.sh -gu'
+ - gefs: 'echo "skipping build"'
+ - gfs: 'echo "skipping build"'
\ No newline at end of file
-- 
GitLab


From b6f012960188c71890f0d2c84da4423662912bd1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 14:36:42 +0000
Subject: [PATCH 390/447] removed throttle option for testing scm

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 26a913f9c..dbc969583 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -60,9 +60,9 @@ pipeline {
                 }
                 stages {
                     stage("build system") {
-                        options {
-                          checkoutToSubdirectory('${HOME}/${system}')
-                        }
+                        //options {
+                        //  checkoutToSubdirectory('${HOME}/${system}')
+                        //}
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-- 
GitLab


From 213f6e450a33e664a20f49586871e1a927b4cecc Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 5 Feb 2024 09:38:47 -0500
Subject: [PATCH 391/447] Update ci/scripts/utils/ci_utils.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ci/scripts/utils/ci_utils.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 7422d3457..1947bf10b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -1,5 +1,15 @@
 #!/bin/env bash
 
+function determine_scheduler() {
+  if command -v sbatch &> /dev/null; then
+    echo "slurm";
+  elif command -v qsub &> /dev/null; then
+    echo "torque";
+  else
+    echo "unknown"
+  fi
+}
+
 function cancel_batch_jobs() {
   # Usage: cancel_batch_jobs <substring>
   # Example: cancel_batch_jobs "C48_ATM_3c4e7f74"
-- 
GitLab


From 2d7e17b9ebfd6035d490a4d83166d61ea724ab9d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 5 Feb 2024 09:40:02 -0500
Subject: [PATCH 392/447] Update ci/scripts/utils/ci_utils.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ci/scripts/utils/ci_utils.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 1947bf10b..38caf2e95 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -21,8 +21,9 @@ function cancel_batch_jobs() {
   local substring=$1
   local job_ids
 
-  # cancel pbs jobs <substring>
-  if [[ ${MACHINE_ID} == "wcoss2" ]]; then
+  scheduler=$(determine_scheduler)
+  case ${scheduler} in;
+    "torque")
     job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do
-- 
GitLab


From 86722b01692a428fa65e5ed87f9762ac70f579e9 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 5 Feb 2024 09:41:53 -0500
Subject: [PATCH 393/447] Update ci/scripts/utils/ci_utils.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ci/scripts/utils/ci_utils.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 38caf2e95..ea5c2396b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -35,7 +35,8 @@ function cancel_batch_jobs() {
       fi
     done
   # cancel slurm jobs <substring>
-  else 
+    ;;
+  "torque")
     job_ids=$(squeue -u "${USER}" -h -o "%i")
 
     for job_id in ${job_ids}; do
-- 
GitLab


From ef1112f475532a678e10978a27500cc54e84c534 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 5 Feb 2024 09:42:13 -0500
Subject: [PATCH 394/447] Update ci/scripts/utils/ci_utils.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ci/scripts/utils/ci_utils.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index ea5c2396b..3ab74e9b5 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -47,7 +47,12 @@ function cancel_batch_jobs() {
         continue
       fi
     done
-  fi
+    ;;
+  *)
+    echo "FATAL: Unknown/unsupported job scheduler"
+    exit 1
+    ;;
+  esac
 }
 
 
-- 
GitLab


From 0448492b71b0b5982b912faf69119a7243108ace Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Mon, 5 Feb 2024 09:43:29 -0500
Subject: [PATCH 395/447] Update modulefiles/module_gwsetup.hera.lua

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 modulefiles/module_gwsetup.hera.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua
index 244a2419e..961403e1a 100644
--- a/modulefiles/module_gwsetup.hera.lua
+++ b/modulefiles/module_gwsetup.hera.lua
@@ -14,6 +14,7 @@ load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
-load(pathJoin("git","2.40.0"))
+local git_ver=os.getenv("git_ver") or "2.40.0"
+load(pathJoin("git", git_ver))
 
 whatis("Description: GFS run setup environment")
-- 
GitLab


From 62fe05435c1c9646675195effa5628bd4ef63e06 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 14:55:15 +0000
Subject: [PATCH 396/447] took out throuttle and put checkout subdir back

---
 Jenkinsfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index dbc969583..491a616a0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,9 +49,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['global_matrix_build'])
-                }
+                //options {
+                //    throttle(['global_matrix_build'])
+                //}
                 axes {
                     axis { 
                         name "system"
@@ -60,9 +60,9 @@ pipeline {
                 }
                 stages {
                     stage("build system") {
-                        //options {
-                        //  checkoutToSubdirectory('${HOME}/${system}')
-                        //}
+                        options {
+                          checkoutToSubdirectory('${HOME}/${system}')
+                        }
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
-- 
GitLab


From 45b87fa62c0d793abd1572af20d11905d8ea39cd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 15:20:30 +0000
Subject: [PATCH 397/447] update build args for faster buld tests

---
 Jenkinsfile               | 3 ++-
 ci/cases/yamls/build.yaml | 6 ++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 491a616a0..f3134ab8c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -115,7 +115,8 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 005db938f..753734ced 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,5 +1,3 @@
 builds:
- #- gefs: './build_all.sh'
- #- gfs: './build_all.sh -gu'
- - gefs: 'echo "skipping build"'
- - gfs: 'echo "skipping build"'
\ No newline at end of file
+ - gefs: './build_all.sh -j 2'
+ - gfs: './build_all.sh -g -j 2'
\ No newline at end of file
-- 
GitLab


From e7ecc8f7cca98c6790a4e910b7e0e7996c62536d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 15:36:02 +0000
Subject: [PATCH 398/447] took stub to create tests and put the action back

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f3134ab8c..5705a12db 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -128,8 +128,7 @@ pipeline {
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "echo 'stub create experiment'", returnStatus: true)
-                                    //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 }
                         }
                     }
-- 
GitLab


From a238eda58e357da09a54221a3a1c514c5792e8b2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 16:17:15 +0000
Subject: [PATCH 399/447] put throttle back and took out -j 2

---
 Jenkinsfile               | 6 +++---
 ci/cases/yamls/build.yaml | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5705a12db..d22d9ea42 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,9 +49,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                //options {
-                //    throttle(['global_matrix_build'])
-                //}
+                options {
+                    throttle(['global_matrix_build'])
+                }
                 axes {
                     axis { 
                         name "system"
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 753734ced..38e1b6913 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,3 @@
 builds:
- - gefs: './build_all.sh -j 2'
- - gfs: './build_all.sh -g -j 2'
\ No newline at end of file
+ - gefs: './build_all.sh'
+ - gfs: './build_all.sh -g'
\ No newline at end of file
-- 
GitLab


From 7099e77f89fdaa70810effbaac858c84e00fa7cb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 16:48:34 +0000
Subject: [PATCH 400/447] put concurrancy back in for the matrixed build stage
 because the scm non reset issue came back

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d22d9ea42..5705a12db 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,9 +49,9 @@ pipeline {
         stage('Build System') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options {
-                    throttle(['global_matrix_build'])
-                }
+                //options {
+                //    throttle(['global_matrix_build'])
+                //}
                 axes {
                     axis { 
                         name "system"
-- 
GitLab


From 25893ae5dfec9516d40d46bef75938bd9bb5a072 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 17:48:47 +0000
Subject: [PATCH 401/447] moved mkdir out of Setup RUNTESTS because it the
 agent options was not using skepDefaultCheckout and erroring on scm GitHub

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5705a12db..aaf058fde 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -37,7 +37,7 @@ pipeline {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
-                    sh( script: "mkdir -p ${HOME}", returnStatus: true)
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     pullRequest.addLabel("CI-${machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
                         pullRequest.removeLabel("CI-${machine}-Ready")
@@ -98,7 +98,6 @@ pipeline {
             options { skipDefaultCheckout() }
             steps {
                 script {
-                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
                          pullRequest.removeLabel("CI-${machine}-Building")
                     }
-- 
GitLab


From 5ce311631fc30e435265b1d69bd608138a72609c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 17:58:29 +0000
Subject: [PATCH 402/447] added explcit workspaces in create and using gfs as
 default for running

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index aaf058fde..3993233df 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -120,7 +120,7 @@ pipeline {
                 }
                 stages {
                     stage('Create Experiment') {
-                        steps {
+                        steps { ws("${HOME}/gfs") {
                                 script {
                                     sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
                                     def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
@@ -128,11 +128,11 @@ pipeline {
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                }
+                                } }
                         }
                     }
                     stage('Run Experiments') {
-                        steps {
+                        steps { ws("${HOME}/gfs") {
                             script {
                                 def HOMEgfs = "${HOME}/gfs"
                                 pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
@@ -144,7 +144,7 @@ pipeline {
                                 pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
                                 error("Failed to run experiments ${Case} on ${machine}")
                                 }
-                            }
+                            } }
                         }
                     }
                 }
-- 
GitLab


From 245a5fb59852edfabda840da35af335a20b040d6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 17:58:38 +0000
Subject: [PATCH 403/447] added explcit workspaces in create and using gfs as
 default for running

---
 Jenkinsfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3993233df..6f1780e07 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -120,15 +120,17 @@ pipeline {
                 }
                 stages {
                     stage('Create Experiment') {
-                        steps { ws("${HOME}/gfs") {
+                        steps { 
                                 script {
                                     sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
                                     def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                } }
+                                    ws(HOMEgfs) {
+                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                    }
+                                } 
                         }
                     }
                     stage('Run Experiments') {
-- 
GitLab


From 0bd51edc7163b1ecc8787975f3be932793d6839f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 18:16:05 +0000
Subject: [PATCH 404/447] took out case and put in if else in cancel_batch_jobs
 (for now)

---
 ci/scripts/utils/ci_utils.sh | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 3ab74e9b5..b0c72a6e9 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -22,8 +22,8 @@ function cancel_batch_jobs() {
   local job_ids
 
   scheduler=$(determine_scheduler)
-  case ${scheduler} in;
-    "torque")
+
+  if [[ "${schduler}" == "torque" ]]; then
     job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do
@@ -34,9 +34,9 @@ function cancel_batch_jobs() {
         continue
       fi
     done
-  # cancel slurm jobs <substring>
-    ;;
-  "torque")
+
+  elif [[ "${scheduler}" == "slurm" ]]; then
+
     job_ids=$(squeue -u "${USER}" -h -o "%i")
 
     for job_id in ${job_ids}; do
@@ -47,13 +47,11 @@ function cancel_batch_jobs() {
         continue
       fi
     done
-    ;;
-  *)
-    echo "FATAL: Unknown/unsupported job scheduler"
-    exit 1
-    ;;
-  esac
-}
+
+  else
+      echo "FATAL: Unknown/unsupported job scheduler"
+      exit 1
+  fi
 
 
 function get_pr_case_list () {
-- 
GitLab


From cc6af4257fa44b16e0710b8ac8b585959c92d133 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 18:38:24 +0000
Subject: [PATCH 405/447] missed bracket at end of cancel_batch_jobs in ci
 utils

---
 ci/scripts/utils/ci_utils.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index b0c72a6e9..6f2426c38 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -52,6 +52,7 @@ function cancel_batch_jobs() {
       echo "FATAL: Unknown/unsupported job scheduler"
       exit 1
   fi
+}
 
 
 function get_pr_case_list () {
-- 
GitLab


From 5c5e860b97a032134ca2e88ec5f3b2c90217cff2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 20:14:10 +0000
Subject: [PATCH 406/447] add true to return status on run-check because it
 through that peski Jobs SCM is not GitHub again and it is the only difference
 to the lines that did not

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6f1780e07..b5228f360 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -128,7 +128,8 @@ pipeline {
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     ws(HOMEgfs) {
-                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                      //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                      sh( script: "echo 'skip create_experiment'" , returnStatus: true)
                                     }
                                 } 
                         }
@@ -140,7 +141,7 @@ pipeline {
                                 pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                 pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                             try {
-                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: false)
+                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                 pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                             } catch (Exception e) {
                                 pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
-- 
GitLab


From 743e8424dc0aa6f0c45cef0e93446d37c209f94b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 20:24:06 +0000
Subject: [PATCH 407/447] update all Jenkins script calls to returnStatus true
 and put full test list back

---
 Jenkinsfile | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b5228f360..4d981d45c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -80,8 +80,8 @@ pipeline {
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
                                         dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: false)
-                                            sh( script: "./link_workflow.sh", returnStatus: false)
+                                            sh( script: "${build_args}", returnStatus: true)
+                                            sh( script: "./link_workflow.sh", returnStatus: true)
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
@@ -114,8 +114,7 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_S2SWA_gefs", "C96_atm3DVar"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -128,8 +127,7 @@ pipeline {
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     ws(HOMEgfs) {
-                                      //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                      sh( script: "echo 'skip create_experiment'" , returnStatus: true)
+                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                     }
                                 } 
                         }
-- 
GitLab


From 01298bb834952ddaa487c10b64363a98b6ab0572 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:28:47 +0000
Subject: [PATCH 408/447] adding with credentials

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4d981d45c..a2ed5a50f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -157,11 +157,13 @@ pipeline {
     post {
         always {
             script {
+                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
                 for (label in pullRequest.labels) {
                     if (label.contains("${machine}")) {
                         pullRequest.removeLabel(label)
                     }
                 }
+                }
             }
         }
         success {
-- 
GitLab


From 0a32c5b0e1cad8b9e9f7cc7edefefd9841a56ae8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:31:15 +0000
Subject: [PATCH 409/447] stub create

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a2ed5a50f..7906c0c75 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -127,7 +127,8 @@ pipeline {
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     ws(HOMEgfs) {
-                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                      //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                      sh( script: "echo 'skip create", returnStatus: true)
                                     }
                                 } 
                         }
-- 
GitLab


From 5bf9d34b35d7d80c206991cfd6f4d41f9c7534d7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:34:20 +0000
Subject: [PATCH 410/447] add short list for exper

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7906c0c75..bfd3dfb90 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -114,7 +114,8 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values  "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
-- 
GitLab


From b74d9826bfe5c510977872f8df5f6c8dd3842842 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:48:25 +0000
Subject: [PATCH 411/447] added withCredntials in all three post sections

---
 Jenkinsfile | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bfd3dfb90..3f2ec815e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -160,31 +160,35 @@ pipeline {
         always {
             script {
                 withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
-                for (label in pullRequest.labels) {
-                    if (label.contains("${machine}")) {
-                        pullRequest.removeLabel(label)
+                    for (label in pullRequest.labels) {
+                        if (label.contains("${machine}")) {
+                            pullRequest.removeLabel(label)
+                        }
                     }
                 }
-                }
             }
         }
         success {
             script {
-                pullRequest.addLabel("CI-${machine}-Passed")
-                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
+                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
+                    pullRequest.addLabel("CI-${machine}-Passed")
+                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                    pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
+                }
             }
         }
         failure {
             script {
-                pullRequest.addLabel("CI-${machine}-Failed")
-                def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-                if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                    def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                        fileContent.eachLine { line ->
-                        if( line.contains(".log")) {
-                            archiveArtifacts artifacts: "${line}", fingerprint: true
+                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
+                    pullRequest.addLabel("CI-${machine}-Failed")
+                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                    pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                    if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                        def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                            fileContent.eachLine { line ->
+                            if( line.contains(".log")) {
+                                archiveArtifacts artifacts: "${line}", fingerprint: true
+                            }
                         }
                     }
                 }
-- 
GitLab


From 04b3ed8138d6ac9b665eaabd0025bb5b96113a2e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:53:43 +0000
Subject: [PATCH 412/447] took out create experments stub

---
 Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3f2ec815e..f44dea693 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -114,8 +114,8 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        //values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values  "C48_S2SWA_gefs", "C96_atm3DVar"
+                        // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -128,8 +128,7 @@ pipeline {
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     ws(HOMEgfs) {
-                                      //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                      sh( script: "echo 'skip create", returnStatus: true)
+                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                     }
                                 } 
                         }
-- 
GitLab


From 0f9605691f0002238bff782048f6d87cc91469c3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 5 Feb 2024 22:56:09 +0000
Subject: [PATCH 413/447] put back in full test list for overnight run

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f44dea693..f905004df 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -114,8 +114,7 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_S2SWA_gefs", "C96_atm3DVar"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
                 stages {
-- 
GitLab


From 271a453b3ae4fc213cf91f5d51911994d19f41b9 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Tue, 6 Feb 2024 00:50:17 -0500
Subject: [PATCH 414/447] Update Jenkinsfile

removed post because withCrednitlals failed on first pass
---
 Jenkinsfile | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f905004df..a182a3827 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -155,29 +155,19 @@ pipeline {
     }
 
     post {
-        always {
-            script {
-                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
-                    for (label in pullRequest.labels) {
-                        if (label.contains("${machine}")) {
-                            pullRequest.removeLabel(label)
-                        }
-                    }
-                }
-            }
-        }
+
         success {
             script {
-                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
+               // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
                     pullRequest.addLabel("CI-${machine}-Passed")
                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                     pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
                 }
-            }
+            //}
         }
         failure {
             script {
-                withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
+                // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
                     pullRequest.addLabel("CI-${machine}-Failed")
                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                     pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
@@ -189,7 +179,7 @@ pipeline {
                             }
                         }
                     }
-                }
+                //}
             }
         }
     }
-- 
GitLab


From 9bd850b94fb164ce1a3ef5129cd360cd4882d089 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 12:47:21 -0500
Subject: [PATCH 415/447] add agent to post block

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a182a3827..5cff08354 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -155,7 +155,7 @@ pipeline {
     }
 
     post {
-
+        agent { label "${MACHINE}-emc" }
         success {
             script {
                // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
-- 
GitLab


From 343c8df9cfdfd96e7ad064bb99075c640a636a7f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 13:37:15 -0500
Subject: [PATCH 416/447] move post into Run Tests block

---
 Jenkinsfile | 67 +++++++++++++++++++++++++++--------------------------
 1 file changed, 34 insertions(+), 33 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5cff08354..ee938d1fe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -110,7 +110,6 @@ pipeline {
         stage('Run Tests') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-                options { skipDefaultCheckout() }
                 axes {
                     axis {
                         name "Case"
@@ -133,7 +132,7 @@ pipeline {
                         }
                     }
                     stage('Run Experiments') {
-                        steps { ws("${HOME}/gfs") {
+                        steps {
                             script {
                                 def HOMEgfs = "${HOME}/gfs"
                                 pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
@@ -145,41 +144,43 @@ pipeline {
                                 pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
                                 error("Failed to run experiments ${Case} on ${machine}")
                                 }
-                            } }
+                            } 
                         }
-                    }
-                }
-            }
-        }
-
-    }
-
-    post {
-        agent { label "${MACHINE}-emc" }
-        success {
-            script {
-               // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
-                    pullRequest.addLabel("CI-${machine}-Passed")
-                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("SUCCESSFULLY ran all CI Cases on ${machine} at ${timestamp}")
-                }
-            //}
-        }
-        failure {
-            script {
-                // withCredentials([usernamePassword(credentialsId: 'emc-bot', usernameVariable: 'GITHUB_USERNAME', passwordVariable: 'GITHUB_PASSWORD')]) {
-                    pullRequest.addLabel("CI-${machine}-Failed")
-                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-                    if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                        def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                            fileContent.eachLine { line ->
-                            if( line.contains(".log")) {
-                                archiveArtifacts artifacts: "${line}", fingerprint: true
+                        post {
+                            always {
+                                script {
+                                    for (label in pullRequest.labels) {
+                                        if (label.contains("${machine}")) {
+                                            pullRequest.removeLabel(label)
+                                        }
+                                    }
+                               }
+                            }
+                            success {
+                                script {
+                                    pullRequest.addLabel("CI-${machine}-Success")
+                                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                                    pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                }
+                            }
+                            failure {
+                                script {
+                                    pullRequest.addLabel("CI-${machine}-Failed")
+                                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                                    pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                                        def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                                        fileContent.eachLine { line ->
+                                            if( line.contains(".log")) {
+                                                archiveArtifacts artifacts: "${line}", fingerprint: true
+                                            }
+                                        }
+                                    }
+                                }
                             }
                         }
                     }
-                //}
+                }
             }
         }
     }
-- 
GitLab


From a57e4af50c58b04edf90787bf267698cbfdcc5ab Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 13:38:54 -0500
Subject: [PATCH 417/447] stubbed out create expermentens again

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ee938d1fe..8ca7e69f8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -125,9 +125,9 @@ pipeline {
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    ws(HOMEgfs) {
-                                      sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                    }
+                                    //ws(HOMEgfs) {
+                                    //  sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                    //}
                                 } 
                         }
                     }
-- 
GitLab


From 967bb6ddb8f1dc9cfe26374367abb4482f73c14e Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 13:45:17 -0500
Subject: [PATCH 418/447] took out sub for create and made case short list

---
 Jenkinsfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ca7e69f8..ca38da181 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -113,7 +113,8 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -125,9 +126,7 @@ pipeline {
                                     system = yaml_case.experiment.system
                                     def HOMEgfs = "${HOME}/${system}"
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    //ws(HOMEgfs) {
-                                    //  sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                    //}
+                                    sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 } 
                         }
                     }
@@ -158,7 +157,7 @@ pipeline {
                             }
                             success {
                                 script {
-                                    pullRequest.addLabel("CI-${machine}-Success")
+                                    pullRequest.addLabel("CI-${machine}-Passed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                                     pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
                                 }
-- 
GitLab


From 0ec0b6f0348d4395d23a54789960d5724ca62852 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 15:22:34 -0500
Subject: [PATCH 419/447] removed setup block

---
 Jenkinsfile | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ca38da181..f73b418e9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,6 +86,10 @@ pipeline {
                                         }
                                     }
                                 }
+                                if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
+                                     pullRequest.removeLabel("CI-${machine}-Building")
+                                }
+                                pullRequest.addLabel("CI-${machine}-Running")
                             }
                         }
                     }
@@ -93,20 +97,6 @@ pipeline {
             }
         }
 
-        stage('Setup RUNTESTS') {
-            agent { label "${MACHINE}-emc" }
-            options { skipDefaultCheckout() }
-            steps {
-                script {
-                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
-                         pullRequest.removeLabel("CI-${machine}-Building")
-                    }
-                    pullRequest.addLabel("CI-${machine}-Running")
-                }
-            }
-
-        }
-
         stage('Run Tests') {
             matrix {
                 agent { label "${MACHINE}-emc" }
-- 
GitLab


From 5914683a69e5dac4d74de271909a109571545cb2 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 15:28:20 -0500
Subject: [PATCH 420/447] just do one test

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f73b418e9..31df1fa1a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -104,7 +104,7 @@ pipeline {
                     axis {
                         name "Case"
                         // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_S2SWA_gefs", "C96_atm3DVar"
+                        values "C48_S2SWA_gefs"
                     }
                 }
                 stages {
-- 
GitLab


From 9b837357e80fa4db6dd86baaf3a9c4d990e4c828 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 15:31:48 -0500
Subject: [PATCH 421/447] added back buildDiscarder

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 31df1fa1a..371ee930a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,6 +7,7 @@ pipeline {
 
     options {
         skipDefaultCheckout()
+        buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
     stages {
-- 
GitLab


From 970e73b2ddc61c9ce51d748cd993e51ce28205ec Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 19:15:54 -0500
Subject: [PATCH 422/447] commented out remove label in build

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 371ee930a..28848291e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -87,9 +87,9 @@ pipeline {
                                         }
                                     }
                                 }
-                                if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
-                                     pullRequest.removeLabel("CI-${machine}-Building")
-                                }
+                                //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
+                                //     pullRequest.removeLabel("CI-${machine}-Building")
+                                //}
                                 pullRequest.addLabel("CI-${machine}-Running")
                             }
                         }
-- 
GitLab


From 23fad4c44659419f40033371bdcd8f91964519ee Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 20:15:29 -0500
Subject: [PATCH 423/447] removed redundent checkout in subdir and commentout
 gitHub access after scm in build

---
 Jenkinsfile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 28848291e..fa22a6ae1 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -61,15 +61,11 @@ pipeline {
                 }
                 stages {
                     stage("build system") {
-                        options {
-                          checkoutToSubdirectory('${HOME}/${system}')
-                        }
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/${system}"
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 ws(HOMEgfs) {
-                                    checkoutToSubdirectory('${HOME}/${system}')
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
@@ -90,7 +86,7 @@ pipeline {
                                 //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
                                 //     pullRequest.removeLabel("CI-${machine}-Building")
                                 //}
-                                pullRequest.addLabel("CI-${machine}-Running")
+                                //pullRequest.addLabel("CI-${machine}-Running")
                             }
                         }
                     }
-- 
GitLab


From 4b8bec7e9d249298a6cf16d042ce13fb19308784 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 22:42:28 -0500
Subject: [PATCH 424/447] added ws with github creditials

---
 Jenkinsfile | 70 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 29 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fa22a6ae1..43610c9a0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,6 +1,7 @@
 def MACHINE = 'none'
 def machine = 'none'
 def HOME = 'none'
+def localworkspace = 'none'
 
 pipeline {
     agent { label 'built-in' }
@@ -16,32 +17,34 @@ pipeline {
             agent { label 'built-in' }
             steps {
                 script {
-                    MACHINE = 'none'
+                    localworkspace = env.WORKSPACE
+                    machine = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
                         if ((label.matches("CI-Hera-Ready"))) {
-                            MACHINE = 'hera'
+                            machine = 'hera'
                         } else if ((label.matches("CI-Orion-Ready"))) {
-                            MACHINE = 'orion'
+                            machine = 'orion'
                         } else if ((label.matches("CI-Hercules-Ready"))) {
-                            MACHINE = 'hercules'
+                            machine = 'hercules'
                         }
                     }
-                    machine = MACHINE[0].toUpperCase() + MACHINE.substring(1)
+                    Machine = machine[0].toUpperCase() + machine.substring(1)
                 }
             }
         }
 
         stage('Get Common Workspace') {
-            agent { label "${MACHINE}-emc" }
+            agent { label "${machine}-emc" }
             steps ( timeout(time: 1, unit: 'HOURS') ) {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
+                    commonworkspace = "${WORKSPCE}"
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    pullRequest.addLabel("CI-${machine}-Building")
-                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Ready") } ) {
-                        pullRequest.removeLabel("CI-${machine}-Ready")
+                    pullRequest.addLabel("CI-${Machine}-Building")
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
+                        pullRequest.removeLabel("CI-${Machine}-Ready")
                     }
                 }
             }
@@ -69,7 +72,7 @@ pipeline {
                                     env.MACHINE_ID = MACHINE
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}")
+                                        ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
                                     } else {
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
@@ -82,11 +85,12 @@ pipeline {
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
+                                
+                                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
+                                         pullRequest.removeLabel("CI-${machine}-Building")
+                                    }
+                                    pullRequest.addLabel("CI-${machine}-Running")
                                 }
-                                //if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
-                                //     pullRequest.removeLabel("CI-${machine}-Building")
-                                //}
-                                //pullRequest.addLabel("CI-${machine}-Running")
                             }
                         }
                     }
@@ -101,7 +105,7 @@ pipeline {
                     axis {
                         name "Case"
                         // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_S2SWA_gefs"
+                        values "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -121,36 +125,43 @@ pipeline {
                         steps {
                             script {
                                 def HOMEgfs = "${HOME}/gfs"
-                                pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-                            try {
-                                sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
-                                pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
-                            } catch (Exception e) {
-                                pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
-                                error("Failed to run experiments ${Case} on ${machine}")
+                                ws(HOMEgfs) {
+                                   pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                   pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                                   try {
+                                      sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
+                                      pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                                    } catch (Exception e) {
+                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
+                                       error("Failed to run experiments ${Case} on ${machine}")
+                                    }
                                 }
                             } 
                         }
                         post {
                             always {
                                 script {
-                                    for (label in pullRequest.labels) {
-                                        if (label.contains("${machine}")) {
-                                            pullRequest.removeLabel(label)
+                                    ws (localworkspace) {
+                                        for (label in pullRequest.labels) {
+                                           if (label.contains("${machine}")) {
+                                               pullRequest.removeLabel(label)
+                                            }
                                         }
                                     }
                                }
                             }
                             success {
                                 script {
-                                    pullRequest.addLabel("CI-${machine}-Passed")
-                                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                    pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    ws (localwospace) {
+                                       pullRequest.addLabel("CI-${machine}-Passed")
+                                       def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                                       pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    }
                                 }
                             }
                             failure {
                                 script {
+                                    ws (localworkspace) {
                                     pullRequest.addLabel("CI-${machine}-Failed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                                     pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
@@ -162,6 +173,7 @@ pipeline {
                                             }
                                         }
                                     }
+                                    }
                                 }
                             }
                         }
-- 
GitLab


From db37fe0a849544652e949657636f2f67094dfe60 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 22:46:13 -0500
Subject: [PATCH 425/447] misspelled WORKSPACE

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 43610c9a0..762387124 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
-                    commonworkspace = "${WORKSPCE}"
+                    commonworkspace = "${WORKSPACE}"
                     sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
                     pullRequest.addLabel("CI-${Machine}-Building")
                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
-- 
GitLab


From 682186dc4652caea02e83ec983fa9b869b602fa0 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Tue, 6 Feb 2024 22:53:48 -0500
Subject: [PATCH 426/447] missed a MACIHNE left in

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 762387124..c4f2c3aa2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,4 +1,4 @@
-def MACHINE = 'none'
+def Machine = 'none'
 def machine = 'none'
 def HOME = 'none'
 def localworkspace = 'none'
@@ -52,7 +52,7 @@ pipeline {
 
         stage('Build System') {
             matrix {
-                agent { label "${MACHINE}-emc" }
+                agent { label "${machine}-emc" }
                 //options {
                 //    throttle(['global_matrix_build'])
                 //}
@@ -69,7 +69,7 @@ pipeline {
                                 def HOMEgfs = "${HOME}/${system}"
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 ws(HOMEgfs) {
-                                    env.MACHINE_ID = MACHINE
+                                    env.MACHINE_ID = machine
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
                                         ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
@@ -100,7 +100,7 @@ pipeline {
 
         stage('Run Tests') {
             matrix {
-                agent { label "${MACHINE}-emc" }
+                agent { label "${machine}-emc" }
                 axes {
                     axis {
                         name "Case"
-- 
GitLab


From 8aa8b026a5a677fdb8524f74c54e9da93e33a3f6 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 09:50:37 -0500
Subject: [PATCH 427/447] took timer out of Get Common

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c4f2c3aa2..3c95a1c91 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,6 +2,7 @@ def Machine = 'none'
 def machine = 'none'
 def HOME = 'none'
 def localworkspace = 'none'
+def commonworkspace = 'none'
 
 pipeline {
     agent { label 'built-in' }
@@ -36,7 +37,7 @@ pipeline {
 
         stage('Get Common Workspace') {
             agent { label "${machine}-emc" }
-            steps ( timeout(time: 1, unit: 'HOURS') ) {
+            steps {
                 script {
                     properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                     HOME = "${WORKSPACE}/TESTDIR"
@@ -72,7 +73,7 @@ pipeline {
                                     env.MACHINE_ID = machine
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
                                         sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        ws(localworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
+                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
                                     } else {
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
-- 
GitLab


From ed0562b58a67f1e28017b0605e27d0edf13f892d Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 10:50:38 -0500
Subject: [PATCH 428/447] updated machine names to Machine in labels missed

---
 Jenkinsfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3c95a1c91..fe71778f6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -87,10 +87,10 @@ pipeline {
                                         }
                                     }
                                 
-                                    if ( pullRequest.labels.any{ value -> value.matches("CI-${machine}-Building") } ) {
-                                         pullRequest.removeLabel("CI-${machine}-Building")
+                                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
+                                         pullRequest.removeLabel("CI-${Machine}-Building")
                                     }
-                                    pullRequest.addLabel("CI-${machine}-Running")
+                                    pullRequest.addLabel("CI-${Machine}-Running")
                                 }
                             }
                         }
@@ -144,7 +144,7 @@ pipeline {
                                 script {
                                     ws (localworkspace) {
                                         for (label in pullRequest.labels) {
-                                           if (label.contains("${machine}")) {
+                                           if (label.contains("${Machine}")) {
                                                pullRequest.removeLabel(label)
                                             }
                                         }
@@ -153,10 +153,10 @@ pipeline {
                             }
                             success {
                                 script {
-                                    ws (localwospace) {
-                                       pullRequest.addLabel("CI-${machine}-Passed")
+                                    ws (localworkspace) {
+                                       pullRequest.addLabel("CI-${Machine}-Passed")
                                        def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                       pullRequest.comment("CI SUCCESS ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                       pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
                                     }
                                 }
                             }
@@ -165,7 +165,7 @@ pipeline {
                                     ws (localworkspace) {
                                     pullRequest.addLabel("CI-${machine}-Failed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                    pullRequest.comment("CI FAILED ${machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
                                     if (fileExists('${HOME}/RUNTESTS/ci.log')) {
                                         def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
                                         fileContent.eachLine { line ->
-- 
GitLab


From d94a873ab4cb0502a0915d6700137e0d71c2997e Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 11:29:22 -0500
Subject: [PATCH 429/447] added commonworkspace in build update labels

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe71778f6..057c40ce3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,11 +86,12 @@ pipeline {
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
-                                
+                                    ws(commonworkspace) {
                                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
                                          pullRequest.removeLabel("CI-${Machine}-Building")
                                     }
                                     pullRequest.addLabel("CI-${Machine}-Running")
+                                    }
                                 }
                             }
                         }
-- 
GitLab


From 5c04148c7798182add5b7201d85bfb7fd206dc3e Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 14:56:59 -0500
Subject: [PATCH 430/447] updated workspace in post to be the same in the
 matrix that ran the scripts and should have a prevouse scm checkout profomed
 in it

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 057c40ce3..7f75a04a0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -132,18 +132,18 @@ pipeline {
                                    pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
-                                      pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                     } catch (Exception e) {
                                        pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
                                        error("Failed to run experiments ${Case} on ${machine}")
                                     }
+                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
                                 }
                             } 
                         }
                         post {
                             always {
                                 script {
-                                    ws (localworkspace) {
+                                    ws (HOMEgfs) {
                                         for (label in pullRequest.labels) {
                                            if (label.contains("${Machine}")) {
                                                pullRequest.removeLabel(label)
@@ -154,7 +154,7 @@ pipeline {
                             }
                             success {
                                 script {
-                                    ws (localworkspace) {
+                                    ws (HOMEgfs) {
                                        pullRequest.addLabel("CI-${Machine}-Passed")
                                        def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                                        pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
@@ -163,7 +163,7 @@ pipeline {
                             }
                             failure {
                                 script {
-                                    ws (localworkspace) {
+                                    ws (HOMEgfs) {
                                     pullRequest.addLabel("CI-${machine}-Failed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                                     pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-- 
GitLab


From 8cf8a2af70b462b9c56acabd5ab3ec66937eec41 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 15:10:06 -0500
Subject: [PATCH 431/447] remove local varible scope on HOMEgfs in Run
 experments in hopes it works in post

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7f75a04a0..3852fbcc9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -117,7 +117,7 @@ pipeline {
                                     sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
                                     def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
                                     system = yaml_case.experiment.system
-                                    def HOMEgfs = "${HOME}/${system}"
+                                    def HOMEgfs = "${HOME}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
                                     env.RUNTESTS = "${HOME}/RUNTESTS"
                                     sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
                                 } 
@@ -126,7 +126,7 @@ pipeline {
                     stage('Run Experiments') {
                         steps {
                             script {
-                                def HOMEgfs = "${HOME}/gfs"
+                                HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
                                    pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
-- 
GitLab


From 196424704910ec60f6161d38574263fc4740be94 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 15:35:27 -0500
Subject: [PATCH 432/447] updated Machine in a label and in PR comments

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3852fbcc9..9c51a7d22 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -129,14 +129,14 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${machine}")
+                                   pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${Machine}")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                     } catch (Exception e) {
-                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${machine}")
-                                       error("Failed to run experiments ${Case} on ${machine}")
+                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}")
+                                       error("Failed to run experiments ${Case} on ${Machine}")
                                     }
-                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${machine}")
+                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}")
                                 }
                             } 
                         }
@@ -164,7 +164,7 @@ pipeline {
                             failure {
                                 script {
                                     ws (HOMEgfs) {
-                                    pullRequest.addLabel("CI-${machine}-Failed")
+                                    pullRequest.addLabel("CI-${Machine}-Failed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
                                     pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
                                     if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-- 
GitLab


From b2c3c316c03d34f53d6fe8b759e4e50861d8db56 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 16:02:18 -0500
Subject: [PATCH 433/447] only replaced commonworkspace with HOMEgfs workspace
 in build matrix

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9c51a7d22..4678299cd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,7 +86,7 @@ pipeline {
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
-                                    ws(commonworkspace) {
+                                    ws(HOMEgfs) {
                                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
                                          pullRequest.removeLabel("CI-${Machine}-Building")
                                     }
-- 
GitLab


From 12ec3627739743ea453ed5270bdc88bf846e254f Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 17:04:23 -0500
Subject: [PATCH 434/447] added info in Running message to use in GitHub PR

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4678299cd..85dbbf285 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -107,7 +107,6 @@ pipeline {
                     axis {
                         name "Case"
                         // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
@@ -129,7 +128,7 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} with pslot ${pslot} on ${Machine}")
+                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                     } catch (Exception e) {
-- 
GitLab


From eb9e48fb3d62465e8cd883bf9509057f316cb327 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 17:10:09 -0500
Subject: [PATCH 435/447] put missing values line back in experments matrix

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 85dbbf285..355a4476f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -107,6 +107,7 @@ pipeline {
                     axis {
                         name "Case"
                         // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values  "C48_S2SWA_gefs", "C96_atm3DVar"
                     }
                 }
                 stages {
-- 
GitLab


From b0dab475c1c2a7639e821a276e76b03e91defef4 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 17:27:29 -0500
Subject: [PATCH 436/447] added u to gfs build and put all tests back in

---
 Jenkinsfile               | 3 +--
 ci/cases/yamls/build.yaml | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 355a4476f..57a09fc2c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -106,8 +106,7 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        // values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                        values  "C48_S2SWA_gefs", "C96_atm3DVar"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
                     }
                 }
                 stages {
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
index 38e1b6913..5398fa188 100644
--- a/ci/cases/yamls/build.yaml
+++ b/ci/cases/yamls/build.yaml
@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -g'
\ No newline at end of file
+ - gfs: './build_all.sh -gu'
\ No newline at end of file
-- 
GitLab


From 9bb5c97173aa9433fc9203ee1d00d4879dcb829b Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 17:56:25 -0500
Subject: [PATCH 437/447] added a stubbed version of the Jenkinsfiel for
 testing

---
 Jenkinsfile_stubbed | 188 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 188 insertions(+)
 create mode 100644 Jenkinsfile_stubbed

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
new file mode 100644
index 000000000..211c87324
--- /dev/null
+++ b/Jenkinsfile_stubbed
@@ -0,0 +1,188 @@
+def Machine = 'none'
+def machine = 'none'
+def HOME = 'none'
+def localworkspace = 'none'
+def commonworkspace = 'none'
+
+pipeline {
+    agent { label 'built-in' }
+
+    options {
+        skipDefaultCheckout()
+        buildDiscarder(logRotator(numToKeepStr: '2'))
+    }
+
+    stages {
+
+        stage('Get Machine') {
+            agent { label 'built-in' }
+            steps {
+                script {
+                    localworkspace = env.WORKSPACE
+                    machine = 'none'
+                    for (label in pullRequest.labels) {
+                        echo "Label: ${label}"
+                        if ((label.matches("CI-Hera-Ready"))) {
+                            machine = 'hera'
+                        } else if ((label.matches("CI-Orion-Ready"))) {
+                            machine = 'orion'
+                        } else if ((label.matches("CI-Hercules-Ready"))) {
+                            machine = 'hercules'
+                        }
+                    }
+                    Machine = machine[0].toUpperCase() + machine.substring(1)
+                }
+            }
+        }
+
+        stage('Get Common Workspace') {
+            agent { label "${machine}-emc" }
+            steps {
+                script {
+                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                    HOME = "${WORKSPACE}/TESTDIR"
+                    commonworkspace = "${WORKSPACE}"
+                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
+                    pullRequest.addLabel("CI-${Machine}-Building")
+                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
+                        pullRequest.removeLabel("CI-${Machine}-Ready")
+                    }
+                }
+            }
+        }
+
+        stage('Build System') {
+            matrix {
+                agent { label "${machine}-emc" }
+                //options {
+                //    throttle(['global_matrix_build'])
+                //}
+                axes {
+                    axis { 
+                        name "system"
+                        values "gfs", "gefs"
+                    }
+                }
+                stages {
+                    stage("build system") {
+                        steps {
+                            script {
+                                def HOMEgfs = "${HOME}/${system}"
+                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
+                                ws(HOMEgfs) {
+                                    env.MACHINE_ID = machine
+                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
+                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
+                                    } else {
+                                        checkout scm
+                                        sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
+                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
+                                        def build_args_list = builds_file['builds']
+                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
+                                        dir("${HOMEgfs}/sorc") {
+                                            //sh( script: "${build_args}", returnStatus: true)
+                                            //sh( script: "./link_workflow.sh", returnStatus: true)
+                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                        }
+                                    }
+                                    ws(HOMEgfs) {
+                                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
+                                         pullRequest.removeLabel("CI-${Machine}-Building")
+                                    }
+                                    pullRequest.addLabel("CI-${Machine}-Running")
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Run Tests') {
+            matrix {
+                agent { label "${machine}-emc" }
+                axes {
+                    axis {
+                        name "Case"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                    }
+                }
+                stages {
+                    stage('Create Experiment') {
+                        steps { 
+                                script {
+                                    sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
+                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
+                                    system = yaml_case.experiment.system
+                                    def HOMEgfs = "${HOME}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                    env.RUNTESTS = "${HOME}/RUNTESTS"
+                                    //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
+                                } 
+                        }
+                    }
+                    stage('Run Experiments') {
+                        steps {
+                            script {
+                                HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
+                                ws(HOMEgfs) {
+                                   //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
+                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}")
+                                   try {
+                                      //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
+                                      sh( script: "echo 'subbed run-check'", returnStatus: true)
+                                    } catch (Exception e) {
+                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}")
+                                       error("Failed to run experiments ${Case} on ${Machine}")
+                                    }
+                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}")
+                                }
+                            } 
+                        }
+                        post {
+                            always {
+                                script {
+                                    ws (HOMEgfs) {
+                                        for (label in pullRequest.labels) {
+                                           if (label.contains("${Machine}")) {
+                                               pullRequest.removeLabel(label)
+                                            }
+                                        }
+                                    }
+                               }
+                            }
+                            success {
+                                script {
+                                    ws (HOMEgfs) {
+                                       pullRequest.addLabel("CI-${Machine}-Passed")
+                                       def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                                       pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    }
+                                }
+                            }
+                            failure {
+                                script {
+                                    ws (HOMEgfs) {
+                                    pullRequest.addLabel("CI-${Machine}-Failed")
+                                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
+                                    pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    if (fileExists('${HOME}/RUNTESTS/ci.log')) {
+                                        def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
+                                        fileContent.eachLine { line ->
+                                            if( line.contains(".log")) {
+                                                archiveArtifacts artifacts: "${line}", fingerprint: true
+                                            }
+                                        }
+                                    }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}
-- 
GitLab


From 44947e092d92a0429c09e337e8faeb2b39fd7284 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 19:34:26 -0500
Subject: [PATCH 438/447] put a try around removelabel in all

---
 Jenkinsfile_stubbed | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
index 211c87324..5d1f23efa 100644
--- a/Jenkinsfile_stubbed
+++ b/Jenkinsfile_stubbed
@@ -146,7 +146,11 @@ pipeline {
                                     ws (HOMEgfs) {
                                         for (label in pullRequest.labels) {
                                            if (label.contains("${Machine}")) {
-                                               pullRequest.removeLabel(label)
+                                               try {
+                                                   pullRequest.removeLabel(label)
+                                               } catch (Exception e) {
+                                                   echo "Failed to remove label ${label}"
+                                               }
                                             }
                                         }
                                     }
-- 
GitLab


From 0f0df6297c7a3a2ebab34b6ef0486e18860cc9ca Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 19:45:15 -0500
Subject: [PATCH 439/447] took pslot out for stubbing

---
 Jenkinsfile_stubbed | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
index 5d1f23efa..efec58b44 100644
--- a/Jenkinsfile_stubbed
+++ b/Jenkinsfile_stubbed
@@ -128,7 +128,7 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}")
+                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS")
                                    try {
                                       //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                       sh( script: "echo 'subbed run-check'", returnStatus: true)
-- 
GitLab


From 562989057ec3bce49713ab6a731bef2ed2f8d792 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 20:07:51 -0500
Subject: [PATCH 440/447] added back just build to stub

---
 Jenkinsfile_stubbed | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
index efec58b44..b64584189 100644
--- a/Jenkinsfile_stubbed
+++ b/Jenkinsfile_stubbed
@@ -71,21 +71,21 @@ pipeline {
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 ws(HOMEgfs) {
                                     env.MACHINE_ID = machine
-                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
-                                    } else {
+                                    //if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
+                                    //    sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
+                                    //    ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
+                                    //} else {
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
                                         dir("${HOMEgfs}/sorc") {
-                                            //sh( script: "${build_args}", returnStatus: true)
-                                            //sh( script: "./link_workflow.sh", returnStatus: true)
+                                            sh( script: "${build_args}", returnStatus: true)
+                                            sh( script: "./link_workflow.sh", returnStatus: true)
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
-                                    }
+                                    //}
                                     ws(HOMEgfs) {
                                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
                                          pullRequest.removeLabel("CI-${Machine}-Building")
-- 
GitLab


From 89954ddc39beddc4cafb152adadda7c959ed80ff Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 21:36:07 -0500
Subject: [PATCH 441/447] removed build non stub and made stub again

---
 Jenkinsfile_stubbed | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
index b64584189..363bb10cc 100644
--- a/Jenkinsfile_stubbed
+++ b/Jenkinsfile_stubbed
@@ -71,21 +71,22 @@ pipeline {
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 ws(HOMEgfs) {
                                     env.MACHINE_ID = machine
-                                    //if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-                                    //    sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-                                    //    ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
-                                    //} else {
+                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
+                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
+                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
+                                    } else {
                                         checkout scm
                                         sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
                                         def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
                                         dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: true)
-                                            sh( script: "./link_workflow.sh", returnStatus: true)
-                                            sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            //sh( script: "${build_args}", returnStatus: true)
+                                            //sh( script: "./link_workflow.sh", returnStatus: true)
+                                            //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
+                                            sh( script: "echo 'stubbing build scripts'", returnStatus: true)
                                         }
-                                    //}
+                                    }
                                     ws(HOMEgfs) {
                                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
                                          pullRequest.removeLabel("CI-${Machine}-Building")
@@ -128,7 +129,7 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS")
+                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/")
                                    try {
                                       //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                       sh( script: "echo 'subbed run-check'", returnStatus: true)
-- 
GitLab


From 1ae8f43b43f16ce424c474ba83fcb1c9c127d2aa Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 21:57:54 -0500
Subject: [PATCH 442/447] added only buils scripts but disabled otherwise scan
 poll

---
 Jenkinsfile_stubbed | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
index 363bb10cc..09fad6675 100644
--- a/Jenkinsfile_stubbed
+++ b/Jenkinsfile_stubbed
@@ -81,10 +81,10 @@ pipeline {
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
                                         dir("${HOMEgfs}/sorc") {
-                                            //sh( script: "${build_args}", returnStatus: true)
-                                            //sh( script: "./link_workflow.sh", returnStatus: true)
+                                            sh( script: "${build_args}", returnStatus: true)
+                                            sh( script: "./link_workflow.sh", returnStatus: true)
                                             //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            sh( script: "echo 'stubbing build scripts'", returnStatus: true)
+                                            sh( script: "echo 'ran with build scripts on'", returnStatus: true)
                                         }
                                     }
                                     ws(HOMEgfs) {
-- 
GitLab


From 3c83d6f15809b1f5974a9cd5996b2b1940e433e9 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Wed, 7 Feb 2024 23:29:21 -0500
Subject: [PATCH 443/447] remove nested ws in build

---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 57a09fc2c..f851b5319 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -86,12 +86,10 @@ pipeline {
                                             sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
                                         }
                                     }
-                                    ws(HOMEgfs) {
                                     if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
                                          pullRequest.removeLabel("CI-${Machine}-Building")
                                     }
                                     pullRequest.addLabel("CI-${Machine}-Running")
-                                    }
                                 }
                             }
                         }
-- 
GitLab


From 2338e9a84d25dc925814eb988de2b9fbd2f65ea5 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Thu, 8 Feb 2024 12:19:33 -0500
Subject: [PATCH 444/447] added comments in Jenkinsfile

---
 Jenkinsfile | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f851b5319..0ada7682d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -12,7 +12,9 @@ pipeline {
         buildDiscarder(logRotator(numToKeepStr: '2'))
     }
 
-    stages {
+    stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
+             // which is used to designate the Nodes in the Jenkins Controler by the agent label
+             // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
 
         stage('Get Machine') {
             agent { label 'built-in' }
@@ -29,7 +31,8 @@ pipeline {
                         } else if ((label.matches("CI-Hercules-Ready"))) {
                             machine = 'hercules'
                         }
-                    }
+                    }         // createing a second machine varible with first letter capital
+                              // because the first letter of the machine name is captitalized in the GitHub labels
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                 }
             }
@@ -67,12 +70,12 @@ pipeline {
                     stage("build system") {
                         steps {
                             script {
-                                def HOMEgfs = "${HOME}/${system}"
+                                def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
                                 sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
                                 ws(HOMEgfs) {
-                                    env.MACHINE_ID = machine
-                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
+                                    env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
+                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
+                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
                                         ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
                                     } else {
                                         checkout scm
@@ -104,7 +107,7 @@ pipeline {
                 axes {
                     axis {
                         name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
+                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" // TODO add dynamic list of cases from env vars (needs addtional plugins)
                     }
                 }
                 stages {
-- 
GitLab


From 62ef2a419ffa15632f3ad56787d674e69c37dffb Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Thu, 8 Feb 2024 14:06:00 -0500
Subject: [PATCH 445/447] added some markdown formatting in GitHub messages

---
 Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0ada7682d..0d655d823 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -129,14 +129,14 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}")
+                                   pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}`")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                     } catch (Exception e) {
-                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}")
+                                       pullRequest.comment("**FAILURE** running experiments: ${Case} on ${Machine}")
                                        error("Failed to run experiments ${Case} on ${Machine}")
                                     }
-                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}")
+                                    pullRequest.comment("**SUCCESS** running experiments: ${Case} on ${Machine}")
                                 }
                             } 
                         }
@@ -157,7 +157,7 @@ pipeline {
                                     ws (HOMEgfs) {
                                        pullRequest.addLabel("CI-${Machine}-Passed")
                                        def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                       pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                       pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
                                     }
                                 }
                             }
@@ -166,7 +166,7 @@ pipeline {
                                     ws (HOMEgfs) {
                                     pullRequest.addLabel("CI-${Machine}-Failed")
                                     def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                    pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
+                                    pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
                                     if (fileExists('${HOME}/RUNTESTS/ci.log')) {
                                         def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
                                         fileContent.eachLine { line ->
-- 
GitLab


From cbc1bbe0caaa10f653ae01ec9913ae91b5e14939 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Thu, 8 Feb 2024 14:18:32 -0500
Subject: [PATCH 446/447] updated some markdown formatting in main GitHub PR
 message

---
 Jenkinsfile         |   2 +-
 Jenkinsfile_stubbed | 193 --------------------------------------------
 2 files changed, 1 insertion(+), 194 deletions(-)
 delete mode 100644 Jenkinsfile_stubbed

diff --git a/Jenkinsfile b/Jenkinsfile
index 0d655d823..0d7c88a96 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -129,7 +129,7 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/${pslot}`")
+                                   pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory `${HOME}/RUNTESTS/${pslot}`")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                     } catch (Exception e) {
diff --git a/Jenkinsfile_stubbed b/Jenkinsfile_stubbed
deleted file mode 100644
index 09fad6675..000000000
--- a/Jenkinsfile_stubbed
+++ /dev/null
@@ -1,193 +0,0 @@
-def Machine = 'none'
-def machine = 'none'
-def HOME = 'none'
-def localworkspace = 'none'
-def commonworkspace = 'none'
-
-pipeline {
-    agent { label 'built-in' }
-
-    options {
-        skipDefaultCheckout()
-        buildDiscarder(logRotator(numToKeepStr: '2'))
-    }
-
-    stages {
-
-        stage('Get Machine') {
-            agent { label 'built-in' }
-            steps {
-                script {
-                    localworkspace = env.WORKSPACE
-                    machine = 'none'
-                    for (label in pullRequest.labels) {
-                        echo "Label: ${label}"
-                        if ((label.matches("CI-Hera-Ready"))) {
-                            machine = 'hera'
-                        } else if ((label.matches("CI-Orion-Ready"))) {
-                            machine = 'orion'
-                        } else if ((label.matches("CI-Hercules-Ready"))) {
-                            machine = 'hercules'
-                        }
-                    }
-                    Machine = machine[0].toUpperCase() + machine.substring(1)
-                }
-            }
-        }
-
-        stage('Get Common Workspace') {
-            agent { label "${machine}-emc" }
-            steps {
-                script {
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    HOME = "${WORKSPACE}/TESTDIR"
-                    commonworkspace = "${WORKSPACE}"
-                    sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
-                    pullRequest.addLabel("CI-${Machine}-Building")
-                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
-                        pullRequest.removeLabel("CI-${Machine}-Ready")
-                    }
-                }
-            }
-        }
-
-        stage('Build System') {
-            matrix {
-                agent { label "${machine}-emc" }
-                //options {
-                //    throttle(['global_matrix_build'])
-                //}
-                axes {
-                    axis { 
-                        name "system"
-                        values "gfs", "gefs"
-                    }
-                }
-                stages {
-                    stage("build system") {
-                        steps {
-                            script {
-                                def HOMEgfs = "${HOME}/${system}"
-                                sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
-                                ws(HOMEgfs) {
-                                    env.MACHINE_ID = machine
-                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) {
-                                        sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim()
-                                        ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
-                                    } else {
-                                        checkout scm
-                                        sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
-                                        def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
-                                        def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
-                                        dir("${HOMEgfs}/sorc") {
-                                            sh( script: "${build_args}", returnStatus: true)
-                                            sh( script: "./link_workflow.sh", returnStatus: true)
-                                            //sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
-                                            sh( script: "echo 'ran with build scripts on'", returnStatus: true)
-                                        }
-                                    }
-                                    ws(HOMEgfs) {
-                                    if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
-                                         pullRequest.removeLabel("CI-${Machine}-Building")
-                                    }
-                                    pullRequest.addLabel("CI-${Machine}-Running")
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        stage('Run Tests') {
-            matrix {
-                agent { label "${machine}-emc" }
-                axes {
-                    axis {
-                        name "Case"
-                        values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar"
-                    }
-                }
-                stages {
-                    stage('Create Experiment') {
-                        steps { 
-                                script {
-                                    sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
-                                    def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
-                                    system = yaml_case.experiment.system
-                                    def HOMEgfs = "${HOME}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                    env.RUNTESTS = "${HOME}/RUNTESTS"
-                                    //sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
-                                } 
-                        }
-                    }
-                    stage('Run Experiments') {
-                        steps {
-                            script {
-                                HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
-                                ws(HOMEgfs) {
-                                   //pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("Running experiments: ${Case} on ${Machine} built against ${system} in directory ${HOMEgfs} with the experiment in directory ${HOME}/RUNTESTS/")
-                                   try {
-                                      //sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
-                                      sh( script: "echo 'subbed run-check'", returnStatus: true)
-                                    } catch (Exception e) {
-                                       pullRequest.comment("FAILURE running experiments: ${Case} on ${Machine}")
-                                       error("Failed to run experiments ${Case} on ${Machine}")
-                                    }
-                                    pullRequest.comment("SUCCESS running experiments: ${Case} on ${Machine}")
-                                }
-                            } 
-                        }
-                        post {
-                            always {
-                                script {
-                                    ws (HOMEgfs) {
-                                        for (label in pullRequest.labels) {
-                                           if (label.contains("${Machine}")) {
-                                               try {
-                                                   pullRequest.removeLabel(label)
-                                               } catch (Exception e) {
-                                                   echo "Failed to remove label ${label}"
-                                               }
-                                            }
-                                        }
-                                    }
-                               }
-                            }
-                            success {
-                                script {
-                                    ws (HOMEgfs) {
-                                       pullRequest.addLabel("CI-${Machine}-Passed")
-                                       def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                       pullRequest.comment("CI SUCCESS ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-                                    }
-                                }
-                            }
-                            failure {
-                                script {
-                                    ws (HOMEgfs) {
-                                    pullRequest.addLabel("CI-${Machine}-Failed")
-                                    def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
-                                    pullRequest.comment("CI FAILED ${Machine} at ${timestamp}\n\nBuilt and ran in directory ${HOME}")
-                                    if (fileExists('${HOME}/RUNTESTS/ci.log')) {
-                                        def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
-                                        fileContent.eachLine { line ->
-                                            if( line.contains(".log")) {
-                                                archiveArtifacts artifacts: "${line}", fingerprint: true
-                                            }
-                                        }
-                                    }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}
-- 
GitLab


From c8c48381c675533b2e71e1a1c246dd4ff67a048b Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Thu, 8 Feb 2024 14:36:23 -0500
Subject: [PATCH 447/447] updated format with one more <br>

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0d7c88a96..c591aae70 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -129,7 +129,7 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                    pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
-                                   pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory `${HOME}/RUNTESTS/${pslot}`")
+                                   pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
                                    try {
                                       sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
                                     } catch (Exception e) {
-- 
GitLab