From 9aa3cc88bd5aa11f12a13d92697de2e1d8bedede Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 17:29:20 +0000
Subject: [PATCH 001/189] trying status of CI as a workflow action
---
.github/workflows/orion_status.yaml | 57 +++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644 .github/workflows/orion_status.yaml
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
new file mode 100644
index 000000000..38eb189b2
--- /dev/null
+++ b/.github/workflows/orion_status.yaml
@@ -0,0 +1,57 @@
+name: OrionStatus
+
+on:
+ workflow_dispatch:
+ pull_request:
+ types: [labeled]
+
+jobs:
+
+ getlabels:
+ runs-on: ubuntu-latest
+ outputs:
+ labels: ${{ steps.id.outputs.labels }}
+ steps:
+ - name: Get Label Steps
+ id: id
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ OWNER: ${{ github.repository_owner }}
+ REPO_NAME: ${{ github.event.repository.name }}
+ PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+ run: |
+ LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
+ LABELS=$(echo "$LABELS1" | tr '\n' ' ')
+ echo "labels=$LABELS" >> $GITHUB_OUTPUT
+
+ Ready:
+ if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready')
+ runs-on: ubuntu-latest
+ needs:
+ - getlabels
+
+ steps:
+ - name: Ready
+ run: echo "Ready"
+
+ Building:
+ if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Building')
+ runs-on: ubuntu-latest
+ needs:
+ - getlabels
+ - Ready
+
+ steps:
+ - name: Building
+ run: echo "Building"
+
+ Passed:
+ if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed')
+ runs-on: ubuntu-latest
+ needs:
+ - getlabels
+ - Building
+
+ steps:
+ - name: Passed
+ run: echo "Passed"
--
GitLab
From 92f2306f340ef297f469451df9ac323df5e8260f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 18:20:45 +0000
Subject: [PATCH 002/189] added PR number as input
---
.github/workflows/orion_status.yaml | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 38eb189b2..a673d5093 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -2,8 +2,11 @@ name: OrionStatus
on:
workflow_dispatch:
- pull_request:
- types: [labeled]
+ inputs:
+ pr_number:
+ description: PR number
+ type: string
+ pull_request: [labeled]
jobs:
@@ -18,7 +21,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OWNER: ${{ github.repository_owner }}
REPO_NAME: ${{ github.event.repository.name }}
- PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+ PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
run: |
LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
LABELS=$(echo "$LABELS1" | tr '\n' ' ')
--
GitLab
From 8c4b2f0246bf0f1709f317ac349d5a43e48ab53b Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 18 Oct 2023 18:24:37 +0000
Subject: [PATCH 003/189] Update orion_status.yaml
type labeled on new line
---
.github/workflows/orion_status.yaml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index a673d5093..210e41fb1 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -6,7 +6,8 @@ on:
pr_number:
description: PR number
type: string
- pull_request: [labeled]
+ pull_request:
+ types: [labeled]
jobs:
--
GitLab
From 74aa7347c606d6a9edcd842f27af8f5c8fb85901 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 18:37:54 +0000
Subject: [PATCH 004/189] take out PR on on
---
.github/workflows/orion_status.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index a673d5093..80bb6861c 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -6,7 +6,6 @@ on:
pr_number:
description: PR number
type: string
- pull_request: [labeled]
jobs:
--
GitLab
From 9b8d071e4d6a550d027719e09d06814673f6992f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 19:07:17 +0000
Subject: [PATCH 005/189] take out PR on on do loops
---
.github/workflows/orion_status.yaml | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 80bb6861c..b92c27031 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -27,17 +27,19 @@ jobs:
echo "labels=$LABELS" >> $GITHUB_OUTPUT
Ready:
- if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready')
runs-on: ubuntu-latest
needs:
- getlabels
steps:
- name: Ready
- run: echo "Ready"
+ run: |
+ until[ $ready ]
+ do
+ ready=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }})
+ done
Building:
- if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Building')
runs-on: ubuntu-latest
needs:
- getlabels
@@ -45,10 +47,13 @@ jobs:
steps:
- name: Building
- run: echo "Building"
+ run: |
+ until[ $building ]
+ do
+ building=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }})
+ done
Passed:
- if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed')
runs-on: ubuntu-latest
needs:
- getlabels
@@ -56,4 +61,8 @@ jobs:
steps:
- name: Passed
- run: echo "Passed"
+ run: |
+ until[ $passed ]
+ do
+ passed=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }})
+ done
--
GitLab
From de4fd80d356447194bc6e9386bc8b77b21dc63db Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 19:15:24 +0000
Subject: [PATCH 006/189] take out PR on on do loops syntax until
---
.github/workflows/orion_status.yaml | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index b92c27031..cc745e4d1 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -34,7 +34,8 @@ jobs:
steps:
- name: Ready
run: |
- until[ $ready ]
+ ready=False
+ until [ $ready ];
do
ready=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }})
done
@@ -48,7 +49,8 @@ jobs:
steps:
- name: Building
run: |
- until[ $building ]
+ building=False
+ until [ $building ];
do
building=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }})
done
@@ -62,7 +64,8 @@ jobs:
steps:
- name: Passed
run: |
- until[ $passed ]
+ passed=False
+ until [ $passed ];
do
passed=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }})
done
--
GitLab
From 3623231113fda22da14de68a0c30d6301619fb78 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 19:32:20 +0000
Subject: [PATCH 007/189] take out PR on on do loops syntax until singleton
---
.github/workflows/orion_status.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index cc745e4d1..86750b1f0 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -35,7 +35,7 @@ jobs:
- name: Ready
run: |
ready=False
- until [ $ready ];
+ until $ready
do
ready=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }})
done
@@ -50,7 +50,7 @@ jobs:
- name: Building
run: |
building=False
- until [ $building ];
+ until $building
do
building=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }})
done
@@ -65,7 +65,7 @@ jobs:
- name: Passed
run: |
passed=False
- until [ $passed ];
+ until $passed
do
passed=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }})
done
--
GitLab
From 218a084cfcef6032419d9df0bd40a1d6840009be Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 19:37:54 +0000
Subject: [PATCH 008/189] take out PR on on do loops syntax until set faulse
---
.github/workflows/orion_status.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 86750b1f0..7c29c23ed 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -34,7 +34,7 @@ jobs:
steps:
- name: Ready
run: |
- ready=False
+ ready=false
until $ready
do
ready=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }})
@@ -49,7 +49,7 @@ jobs:
steps:
- name: Building
run: |
- building=False
+ building=false
until $building
do
building=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }})
@@ -64,7 +64,7 @@ jobs:
steps:
- name: Passed
run: |
- passed=False
+ passed=false
until $passed
do
passed=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }})
--
GitLab
From 83d5c791058b0486c7d506f88d28fb2303ad113d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 19:53:11 +0000
Subject: [PATCH 009/189] take out PR on on do loops syntax until direct
---
.github/workflows/orion_status.yaml | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 7c29c23ed..743d89b9f 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -34,10 +34,8 @@ jobs:
steps:
- name: Ready
run: |
- ready=false
- until $ready
+ until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
do
- ready=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }})
done
Building:
@@ -49,10 +47,8 @@ jobs:
steps:
- name: Building
run: |
- building=false
- until $building
+ until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
do
- building=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }})
done
Passed:
@@ -64,8 +60,6 @@ jobs:
steps:
- name: Passed
run: |
- passed=false
- until $passed
+ until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
do
- passed=$(${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }})
done
--
GitLab
From de55f1a651e70204e0164fc5ef00f9faf6aaa9de Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 20:01:32 +0000
Subject: [PATCH 010/189] take out PR on on do loops needed noop
---
.github/workflows/orion_status.yaml | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 743d89b9f..e5d62e19f 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -36,7 +36,8 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
do
- done
+ noop=""
+ done
Building:
runs-on: ubuntu-latest
@@ -49,7 +50,8 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
do
- done
+ noop=""
+ done
Passed:
runs-on: ubuntu-latest
@@ -62,4 +64,5 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
do
- done
+ noop=""
+ done
--
GitLab
From 5b3e6cc3a6b3b3b44a9e12032c866d6137749f27 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 21:06:49 +0000
Subject: [PATCH 011/189] take out PR jobs with no steps
---
.github/workflows/orion_status.yaml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index e5d62e19f..eacfa374d 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -31,7 +31,6 @@ jobs:
needs:
- getlabels
- steps:
- name: Ready
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
@@ -45,7 +44,6 @@ jobs:
- getlabels
- Ready
- steps:
- name: Building
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
@@ -59,7 +57,6 @@ jobs:
- getlabels
- Building
- steps:
- name: Passed
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
--
GitLab
From 022dd282373d24e702ef059948b65a1cd89144b8 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 21:07:44 +0000
Subject: [PATCH 012/189] take out PR jobs need steps
---
.github/workflows/orion_status.yaml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index eacfa374d..cc483383d 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -31,6 +31,7 @@ jobs:
needs:
- getlabels
+ steps:
- name: Ready
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
@@ -44,6 +45,7 @@ jobs:
- getlabels
- Ready
+ steps:
- name: Building
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
@@ -57,9 +59,10 @@ jobs:
- getlabels
- Building
+ steps:
- name: Passed
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
do
- noop=""
+ noop=""
done
--
GitLab
From c9c8ae25820be9f8ca8a29d37e360d7a0845005f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Wed, 18 Oct 2023 21:09:01 +0000
Subject: [PATCH 013/189] take out PR jobs need steps
---
.github/workflows/orion_status.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index cc483383d..5c53186e5 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -36,7 +36,7 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
do
- noop=""
+ sleep 10m
done
Building:
@@ -50,7 +50,7 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
do
- noop=""
+ sleep 10m
done
Passed:
@@ -64,5 +64,5 @@ jobs:
run: |
until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
do
- noop=""
+ sleep 10m
done
--
GitLab
From 74a5b5a85b6fc3667c621cf9ff757f2f6dbca14e Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Thu, 19 Oct 2023 19:06:17 +0000
Subject: [PATCH 014/189] added reusable workflow for getting lables for each
job
---
.github/workflows/getlabels.yaml | 57 +++++++++++++++++++++++++++++
.github/workflows/orion_status.yaml | 17 ---------
2 files changed, 57 insertions(+), 17 deletions(-)
create mode 100644 .github/workflows/getlabels.yaml
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
new file mode 100644
index 000000000..dda30b218
--- /dev/null
+++ b/.github/workflows/getlabels.yaml
@@ -0,0 +1,57 @@
+on:
+ workflow_call:
+ inputs:
+ pr_number:
+ required: true
+ type: string
+ max_runtime:
+ required: false
+ type: string
+ stage:
+ required: true
+ type: string
+
+ outputs:
+ state:
+ description: "The return state of Stage (true/false)"
+ value: ${{ jobs.getlabels.outputs.state }}
+
+ secrets:
+ token:
+ required: true
+jobs:
+
+ getlabels:
+ runs-on: ubuntu-latest
+ outputs:
+ state: ${{ steps.id.outputs.state }}
+ steps:
+ - name: Get Label Steps
+ id: id
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ OWNER: ${{ github.repository_owner }}
+ REPO_NAME: ${{ github.event.repository.name }}
+ PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
+ STAGE: ${{ inputs.stage }}
+ MAX_TIME: ${{ inputs.max_runtime }}
+ run: |
+ DONE=false
+ count=0
+ until false
+ do
+ LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
+ LABELS=$(echo "$LABELS1" | tr '\n' ' ')
+ check_label="CI-Orion-${STAGE}"
+ if [[ "${LABELS}" == *"${check_label}"* ]]; then
+ DONE=true
+ break
+ fi
+ sleep 10m
+ count=$((count+10))
+ if [[ ${count} -gt ${MAX_TIME} ]]; then
+ DONE=false
+ break
+ fi
+ done
+ echo "state=$DONE" >> $GITHUB_OUPUT
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 5c53186e5..880060950 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -9,23 +9,6 @@ on:
jobs:
- getlabels:
- runs-on: ubuntu-latest
- outputs:
- labels: ${{ steps.id.outputs.labels }}
- steps:
- - name: Get Label Steps
- id: id
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- OWNER: ${{ github.repository_owner }}
- REPO_NAME: ${{ github.event.repository.name }}
- PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
- run: |
- LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
- LABELS=$(echo "$LABELS1" | tr '\n' ' ')
- echo "labels=$LABELS" >> $GITHUB_OUTPUT
-
Ready:
runs-on: ubuntu-latest
needs:
--
GitLab
From 9e306ef382b4844d78e28e5ded0d8212661482f8 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Thu, 19 Oct 2023 19:14:11 +0000
Subject: [PATCH 015/189] fixed uses path
---
.github/workflows/orion_status.yaml | 78 +++++++++++++++++------------
1 file changed, 47 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 880060950..d56a05012 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -6,46 +6,62 @@ on:
pr_number:
description: PR number
type: string
+ max_runtime:
+ description: Maximum time out time for running experment
+ type: string
jobs:
+ get_Ready:
+ uses: ./.github/workflows/getlabels.yaml
+ with:
+ pr_number: inputs.pr_number
+ max_runtime: 20
+ stage: "Ready"
+ secrets: inherit
+
Ready:
runs-on: ubuntu-latest
- needs:
- - getlabels
-
+ needs: get_Ready
steps:
- - name: Ready
- run: |
- until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Ready') }}
- do
- sleep 10m
- done
+ - run: |
+ if [[ ${{ needs.get_Ready.outputs.state }} ]]; then
+ echo "Waiting for Ready State timmed out"
+ exit 1
+ fi
+
+ get_Building:
+ uses: ./.github/workflows/getlabels.yaml
+ with:
+ pr_number: inputs.pr_number
+ max_runtime: 40
+ stage: "Building"
+ secrets: inherit
Building:
runs-on: ubuntu-latest
- needs:
- - getlabels
- - Ready
-
+ needs: get_Building
steps:
- - name: Building
- run: |
- until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Building') }}
- do
- sleep 10m
- done
-
- Passed:
- runs-on: ubuntu-latest
- needs:
- - getlabels
- - Building
+ - run: |
+ if [[ ${{ needs.get_Building.outputs.state }} ]]; then
+ echo "Waiting for Ready State timmed out"
+ exit 1
+ fi
+ get_Running:
+ uses: ./.github/workflows/getlabels.yaml
+ with:
+ pr_number: inputs.pr_number
+ max_runtime: 60
+ stage: "Running"
+ secrets: inherit
+
+ Running:
+ runs-on: ubuntu-latest
+ needs: get_Running
steps:
- - name: Passed
- run: |
- until ${{ contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') }}
- do
- sleep 10m
- done
+ - run: |
+ if [[ ${{ needs.get_Running.outputs.state }} ]]; then
+ echo "Waiting for Ready State timmed out"
+ exit 1
+ fi
\ No newline at end of file
--
GitLab
From 1430ab38082525044b85286b51d8584aedbf63b3 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Thu, 19 Oct 2023 19:39:17 +0000
Subject: [PATCH 016/189] added dependances for main jobs and name in getlabels
---
.github/workflows/getlabels.yaml | 2 ++
.github/workflows/orion_status.yaml | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index dda30b218..ffc40cc92 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -1,3 +1,5 @@
+name: Get Labels
+
on:
workflow_call:
inputs:
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index d56a05012..b2a80a40b 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -40,7 +40,7 @@ jobs:
Building:
runs-on: ubuntu-latest
- needs: get_Building
+ needs: [get_Building, Ready]
steps:
- run: |
if [[ ${{ needs.get_Building.outputs.state }} ]]; then
@@ -58,7 +58,7 @@ jobs:
Running:
runs-on: ubuntu-latest
- needs: get_Running
+ needs: [get_Running, Building]
steps:
- run: |
if [[ ${{ needs.get_Running.outputs.state }} ]]; then
--
GitLab
From 56b07342064b4d4d3404180241002368858191f8 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Thu, 19 Oct 2023 20:06:10 +0000
Subject: [PATCH 017/189] change order in needs
---
.github/workflows/orion_status.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index b2a80a40b..7cc7a1252 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -40,7 +40,7 @@ jobs:
Building:
runs-on: ubuntu-latest
- needs: [get_Building, Ready]
+ needs: [Ready, get_Building]
steps:
- run: |
if [[ ${{ needs.get_Building.outputs.state }} ]]; then
@@ -58,7 +58,7 @@ jobs:
Running:
runs-on: ubuntu-latest
- needs: [get_Running, Building]
+ needs: [Building, get_Running]
steps:
- run: |
if [[ ${{ needs.get_Running.outputs.state }} ]]; then
--
GitLab
From 7e14adb123904628109c64c7d288c9a49d8f15c2 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:27:34 +0000
Subject: [PATCH 018/189] took out poll
---
.github/workflows/getlabels.yaml | 25 +++++++------------------
1 file changed, 7 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index ffc40cc92..c9dcbc615 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -38,22 +38,11 @@ jobs:
STAGE: ${{ inputs.stage }}
MAX_TIME: ${{ inputs.max_runtime }}
run: |
- DONE=false
- count=0
- until false
- do
- LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
- LABELS=$(echo "$LABELS1" | tr '\n' ' ')
- check_label="CI-Orion-${STAGE}"
- if [[ "${LABELS}" == *"${check_label}"* ]]; then
- DONE=true
- break
- fi
- sleep 10m
- count=$((count+10))
- if [[ ${count} -gt ${MAX_TIME} ]]; then
- DONE=false
- break
- fi
- done
+ LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
+ LABELS=$(echo "$LABELS1" | tr '\n' ' ')
+ check_label="CI-Orion-${STAGE}"
+ if [[ "${LABELS}" == *"${check_label}"* ]]; then
+ DONE=true
+ break
+ fi
echo "state=$DONE" >> $GITHUB_OUPUT
--
GitLab
From 0615fad5eb03f5e743ddd8b7d8ab1154c6e5fe8c Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:31:03 +0000
Subject: [PATCH 019/189] took out secrets required line
---
.github/workflows/getlabels.yaml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index c9dcbc615..156dcc332 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -18,9 +18,6 @@ on:
description: "The return state of Stage (true/false)"
value: ${{ jobs.getlabels.outputs.state }}
- secrets:
- token:
- required: true
jobs:
getlabels:
--
GitLab
From a9145c79014be6ac4674067de7aa9920d6c762c8 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:33:39 +0000
Subject: [PATCH 020/189] took out secrets required line
---
.github/workflows/getlabels.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 156dcc332..3c5f44147 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -35,6 +35,7 @@ jobs:
STAGE: ${{ inputs.stage }}
MAX_TIME: ${{ inputs.max_runtime }}
run: |
+ echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
LABELS=$(echo "$LABELS1" | tr '\n' ' ')
check_label="CI-Orion-${STAGE}"
--
GitLab
From 9b8d608932c135256ad1f32ffea10baffd82f180 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:37:40 +0000
Subject: [PATCH 021/189] added {{}} on inputs.pr_number
---
.github/workflows/orion_status.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 7cc7a1252..3bcd3cb5c 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -15,7 +15,7 @@ jobs:
get_Ready:
uses: ./.github/workflows/getlabels.yaml
with:
- pr_number: inputs.pr_number
+ pr_number: ${{ inputs.pr_number }}
max_runtime: 20
stage: "Ready"
secrets: inherit
--
GitLab
From ac1b51cc79271c5e6baa0bef4586e64efd03e4f9 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:42:21 +0000
Subject: [PATCH 022/189] fixed other pr_number assiginments
---
.github/workflows/getlabels.yaml | 27 +++++++++++++++++++--------
.github/workflows/orion_status.yaml | 4 ++--
2 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 3c5f44147..3d6bfaffa 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -35,12 +35,23 @@ jobs:
STAGE: ${{ inputs.stage }}
MAX_TIME: ${{ inputs.max_runtime }}
run: |
- echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
- LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
- LABELS=$(echo "$LABELS1" | tr '\n' ' ')
- check_label="CI-Orion-${STAGE}"
- if [[ "${LABELS}" == *"${check_label}"* ]]; then
- DONE=true
- break
- fi
+ DONE=false
+ count=0
+ until false
+ do
+ echo "$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
+ LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
+ LABELS=$(echo "$LABELS1" | tr '\n' ' ')
+ check_label="CI-Orion-${STAGE}"
+ if [[ "${LABELS}" == *"${check_label}"* ]]; then
+ DONE=true
+ break
+ fi
+ sleep 10m
+ count=$((count+10))
+ if [[ ${count} -gt ${MAX_TIME} ]]; then
+ DONE=false
+ break
+ fi
+ done
echo "state=$DONE" >> $GITHUB_OUPUT
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 3bcd3cb5c..e33141173 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -33,7 +33,7 @@ jobs:
get_Building:
uses: ./.github/workflows/getlabels.yaml
with:
- pr_number: inputs.pr_number
+ pr_number: ${{ inputs.pr_number }}
max_runtime: 40
stage: "Building"
secrets: inherit
@@ -51,7 +51,7 @@ jobs:
get_Running:
uses: ./.github/workflows/getlabels.yaml
with:
- pr_number: inputs.pr_number
+ pr_number: ${{ inputs.pr_number }}
max_runtime: 60
stage: "Running"
secrets: inherit
--
GitLab
From 404a4543078bfd6cdbf7d90a3f0afb90d6196e6d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:43:48 +0000
Subject: [PATCH 023/189] EOF error working
---
.github/workflows/getlabels.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 3d6bfaffa..993d5a715 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,4 @@ jobs:
break
fi
done
- echo "state=$DONE" >> $GITHUB_OUPUT
+ echo "state=$DONE" >> $GITHUB_OUPUT
--
GitLab
From 381832a197fde0e4ad63a120c9017e8d45f164fa Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:45:43 +0000
Subject: [PATCH 024/189] EOF error working II
---
.github/workflows/getlabels.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 993d5a715..2372bbb9a 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -39,7 +39,7 @@ jobs:
count=0
until false
do
- echo "$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
+ echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
LABELS=$(echo "$LABELS1" | tr '\n' ' ')
check_label="CI-Orion-${STAGE}"
--
GitLab
From 3f6bd46d73ec96621ee178abe40f5a97cd068f49 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 01:52:33 +0000
Subject: [PATCH 025/189] ambiguous redrirect to GITHUB_OUTPUT
---
.github/workflows/getlabels.yaml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 2372bbb9a..020e937e4 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,5 @@ jobs:
break
fi
done
- echo "state=$DONE" >> $GITHUB_OUPUT
+ echo "state=$DONE"
+ echo "state=$DONE" >> $env:GITHUB_OUPUT
--
GitLab
From ab5d110b795c758578ed9d4be63c915728c5ed77 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 02:13:02 +0000
Subject: [PATCH 026/189] tryig to get needs output from get_Ready
---
.github/workflows/getlabels.yaml | 9 ++++-----
.github/workflows/orion_status.yaml | 5 +++--
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 020e937e4..15fc204c2 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -2,6 +2,10 @@ name: Get Labels
on:
workflow_call:
+ outputs:
+ state:
+ description: "The return state of Stage (true/false)"
+ value: ${{ jobs.getlabels.outputs.state }}
inputs:
pr_number:
required: true
@@ -13,11 +17,6 @@ on:
required: true
type: string
- outputs:
- state:
- description: "The return state of Stage (true/false)"
- value: ${{ jobs.getlabels.outputs.state }}
-
jobs:
getlabels:
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index e33141173..b3c76a34d 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -30,8 +30,9 @@ jobs:
exit 1
fi
- get_Building:
+ get_Building:
uses: ./.github/workflows/getlabels.yaml
+ needs: Ready
with:
pr_number: ${{ inputs.pr_number }}
max_runtime: 40
@@ -40,7 +41,7 @@ jobs:
Building:
runs-on: ubuntu-latest
- needs: [Ready, get_Building]
+ needs: get_Building
steps:
- run: |
if [[ ${{ needs.get_Building.outputs.state }} ]]; then
--
GitLab
From 35c4877f2a953035a9fcd25f8f5328d1b3034db2 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 02:26:54 +0000
Subject: [PATCH 027/189] still not getting outputs
---
.github/workflows/getlabels.yaml | 2 +-
.github/workflows/orion_status.yaml | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 15fc204c2..8789da3cd 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,4 @@ jobs:
fi
done
echo "state=$DONE"
- echo "state=$DONE" >> $env:GITHUB_OUPUT
+ echo "state=$DONE" >> "${GITHUB_OUPUT}"
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index b3c76a34d..d607f1725 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -51,6 +51,7 @@ jobs:
get_Running:
uses: ./.github/workflows/getlabels.yaml
+ needs: Building
with:
pr_number: ${{ inputs.pr_number }}
max_runtime: 60
@@ -59,7 +60,7 @@ jobs:
Running:
runs-on: ubuntu-latest
- needs: [Building, get_Running]
+ needs: get_Running
steps:
- run: |
if [[ ${{ needs.get_Running.outputs.state }} ]]; then
--
GitLab
From 725e5258559a8b6a0f6d95ff592aa31315656170 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 02:34:03 +0000
Subject: [PATCH 028/189] quotes on GITHUB_OUTPUT
---
.github/workflows/getlabels.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 8789da3cd..7f9e1c090 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,4 @@ jobs:
fi
done
echo "state=$DONE"
- echo "state=$DONE" >> "${GITHUB_OUPUT}"
+ echo "state=$DONE" >> ${GITHUB_OUPUT}
--
GitLab
From 461297d10f29a920b5c5e4b9394d0de8a40f38e9 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 02:49:06 +0000
Subject: [PATCH 029/189] trying crazy Sys IO
---
.github/workflows/getlabels.yaml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 7f9e1c090..c8193482e 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,7 @@ jobs:
fi
done
echo "state=$DONE"
- echo "state=$DONE" >> ${GITHUB_OUPUT}
+ var file = Environment.GetEnvironmentVariable("GITHUB_OUTPUT");
+ var content = System.IO.File.ReadAllText(file);
+ System.IO.File.WriteAllText(file, content + "state=${DONE}" + "\n");
+ # echo "state=$DONE" >> ${GITHUB_OUPUT}
--
GitLab
From 9bdf6f51c443bfd3a2249c54e4d0599baed27728 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:07:41 +0000
Subject: [PATCH 030/189] addeing yml file
---
.github/workflows/getlabels.yml | 57 +++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644 .github/workflows/getlabels.yml
diff --git a/.github/workflows/getlabels.yml b/.github/workflows/getlabels.yml
new file mode 100644
index 000000000..e8ebc733f
--- /dev/null
+++ b/.github/workflows/getlabels.yml
@@ -0,0 +1,57 @@
+name: Get Labels
+
+on:
+ workflow_call:
+ outputs:
+ state:
+ description: "The return state of Stage (true/false)"
+ value: ${{ jobs.getlabels.outputs.state }}
+ inputs:
+ pr_number:
+ required: true
+ type: string
+ max_runtime:
+ required: false
+ type: string
+ stage:
+ required: true
+ type: string
+
+jobs:
+
+ getlabels:
+ runs-on: ubuntu-latest
+ outputs:
+ state: ${{ steps.id.outputs.state }}
+ steps:
+ - name: Get Label Steps
+ id: id
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ OWNER: ${{ github.repository_owner }}
+ REPO_NAME: ${{ github.event.repository.name }}
+ PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
+ STAGE: ${{ inputs.stage }}
+ MAX_TIME: ${{ inputs.max_runtime }}
+ run: |
+ DONE=false
+ count=0
+ until false
+ do
+ echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
+ LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
+ LABELS=$(echo "$LABELS1" | tr '\n' ' ')
+ check_label="CI-Orion-${STAGE}"
+ if [[ "${LABELS}" == *"${check_label}"* ]]; then
+ DONE=true
+ break
+ fi
+ sleep 10m
+ count=$((count+10))
+ if [[ ${count} -gt ${MAX_TIME} ]]; then
+ DONE=false
+ break
+ fi
+ done
+ echo "state=${DONE}"
+ echo "state=${DONE}" >> ${GITHUB_OUPUT}
--
GitLab
From 3e7a55982e36e55f30cf57deade49e036c9fae36 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:08:22 +0000
Subject: [PATCH 031/189] remove yml file
---
.github/workflows/getlabels.yml | 57 ---------------------------------
1 file changed, 57 deletions(-)
delete mode 100644 .github/workflows/getlabels.yml
diff --git a/.github/workflows/getlabels.yml b/.github/workflows/getlabels.yml
deleted file mode 100644
index e8ebc733f..000000000
--- a/.github/workflows/getlabels.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-name: Get Labels
-
-on:
- workflow_call:
- outputs:
- state:
- description: "The return state of Stage (true/false)"
- value: ${{ jobs.getlabels.outputs.state }}
- inputs:
- pr_number:
- required: true
- type: string
- max_runtime:
- required: false
- type: string
- stage:
- required: true
- type: string
-
-jobs:
-
- getlabels:
- runs-on: ubuntu-latest
- outputs:
- state: ${{ steps.id.outputs.state }}
- steps:
- - name: Get Label Steps
- id: id
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- OWNER: ${{ github.repository_owner }}
- REPO_NAME: ${{ github.event.repository.name }}
- PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
- STAGE: ${{ inputs.stage }}
- MAX_TIME: ${{ inputs.max_runtime }}
- run: |
- DONE=false
- count=0
- until false
- do
- echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
- LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
- LABELS=$(echo "$LABELS1" | tr '\n' ' ')
- check_label="CI-Orion-${STAGE}"
- if [[ "${LABELS}" == *"${check_label}"* ]]; then
- DONE=true
- break
- fi
- sleep 10m
- count=$((count+10))
- if [[ ${count} -gt ${MAX_TIME} ]]; then
- DONE=false
- break
- fi
- done
- echo "state=${DONE}"
- echo "state=${DONE}" >> ${GITHUB_OUPUT}
--
GitLab
From b6566149215f90e4afb000d6390b60754185ed00 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:09:13 +0000
Subject: [PATCH 032/189] added () on DONE
---
.github/workflows/getlabels.yaml | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index c8193482e..e8ebc733f 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -53,8 +53,5 @@ jobs:
break
fi
done
- echo "state=$DONE"
- var file = Environment.GetEnvironmentVariable("GITHUB_OUTPUT");
- var content = System.IO.File.ReadAllText(file);
- System.IO.File.WriteAllText(file, content + "state=${DONE}" + "\n");
- # echo "state=$DONE" >> ${GITHUB_OUPUT}
+ echo "state=${DONE}"
+ echo "state=${DONE}" >> ${GITHUB_OUPUT}
--
GitLab
From ed3429e3f3c1cf4a0258b0d7f14c256ef71e1c3c Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:16:05 +0000
Subject: [PATCH 033/189] adding env back to GITHUB_OUPUT
---
.github/workflows/getlabels.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index e8ebc733f..b861658a8 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -54,4 +54,4 @@ jobs:
fi
done
echo "state=${DONE}"
- echo "state=${DONE}" >> ${GITHUB_OUPUT}
+ echo "state=${DONE}" >> $env:GITHUB_OUTPUT
--
GitLab
From 00e92e286387b4bca03285420a93b47ec1844f31 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:24:48 +0000
Subject: [PATCH 034/189] added read write permissions to reuse
---
.github/workflows/getlabels.yaml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index b861658a8..156ee222a 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -17,6 +17,10 @@ on:
required: true
type: string
+permissions:
+ contents: read
+ packages: write
+
jobs:
getlabels:
@@ -54,4 +58,4 @@ jobs:
fi
done
echo "state=${DONE}"
- echo "state=${DONE}" >> $env:GITHUB_OUTPUT
+ echo "state=${DONE}" >> $GITHUB_OUTPUT
--
GitLab
From 7cda4501f36017947fe8564011c9cde106e38ee0 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:34:14 +0000
Subject: [PATCH 035/189] fixed logic in status workflow for on set
---
.github/workflows/orion_status.yaml | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index d607f1725..3be30a623 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -13,7 +13,7 @@ on:
jobs:
get_Ready:
- uses: ./.github/workflows/getlabels.yaml
+ uses: ./.github/workflows/getlabels.yml
with:
pr_number: ${{ inputs.pr_number }}
max_runtime: 20
@@ -25,9 +25,11 @@ jobs:
needs: get_Ready
steps:
- run: |
- if [[ ${{ needs.get_Ready.outputs.state }} ]]; then
- echo "Waiting for Ready State timmed out"
+ if [[ "${{ needs.get_Ready.outputs.state }}" == "false" ]]; then
+ echo "Ready Timmed out"
exit 1
+ elif [[ "${{ needs.get_Ready.outputs.state }}" == "true" ]]; then
+ echo "Ready Set"
fi
get_Building:
@@ -44,10 +46,12 @@ jobs:
needs: get_Building
steps:
- run: |
- if [[ ${{ needs.get_Building.outputs.state }} ]]; then
- echo "Waiting for Ready State timmed out"
+ if [[ "${{ needs.get_Building.outputs.state }}" == "false" ]]; then
+ echo "Building Timmed out"
exit 1
- fi
+ elif [[ "${{ needs.get_Building.outputs.state }}" == "true" ]]; then
+ echo "Building Set"
+ fi
get_Running:
uses: ./.github/workflows/getlabels.yaml
@@ -63,7 +67,9 @@ jobs:
needs: get_Running
steps:
- run: |
- if [[ ${{ needs.get_Running.outputs.state }} ]]; then
- echo "Waiting for Ready State timmed out"
+ if [[ "${{ needs.get_Running.outputs.state }}" == "false" ]]; then
+ echo "Running Timmed out"
exit 1
+ elif [[ "${{ needs.get_Running.outputs.state }}" == "true" ]]; then
+ echo "Running Set"
fi
\ No newline at end of file
--
GitLab
From 522e4e30d4f224e4cc81cff42ef29ac699b784cf Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:37:30 +0000
Subject: [PATCH 036/189] noop on getlables
---
.github/workflows/getlabels.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
index 156ee222a..cff128217 100644
--- a/.github/workflows/getlabels.yaml
+++ b/.github/workflows/getlabels.yaml
@@ -16,7 +16,6 @@ on:
stage:
required: true
type: string
-
permissions:
contents: read
packages: write
--
GitLab
From 752210cfed4f4e0b50f7e5123c70b9b000745348 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:39:26 +0000
Subject: [PATCH 037/189] issue with orion status
---
.github/workflows/orion_status.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index 3be30a623..a9ce1def1 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -25,10 +25,10 @@ jobs:
needs: get_Ready
steps:
- run: |
- if [[ "${{ needs.get_Ready.outputs.state }}" == "false" ]]; then
+ if [[ ${{ needs.get_Ready.outputs.state }} == "false" ]]; then
echo "Ready Timmed out"
exit 1
- elif [[ "${{ needs.get_Ready.outputs.state }}" == "true" ]]; then
+ elif [[ ${{ needs.get_Ready.outputs.state }} == "true" ]]; then
echo "Ready Set"
fi
--
GitLab
From 56b6a5f2d2bf8c84b53739fe64afb443b6ae76b7 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terry.mcguinness@noaa.gov>
Date: Sat, 21 Oct 2023 03:42:10 +0000
Subject: [PATCH 038/189] misspelled yaml as yml
---
.github/workflows/orion_status.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
index a9ce1def1..76d557127 100644
--- a/.github/workflows/orion_status.yaml
+++ b/.github/workflows/orion_status.yaml
@@ -13,7 +13,7 @@ on:
jobs:
get_Ready:
- uses: ./.github/workflows/getlabels.yml
+ uses: ./.github/workflows/getlabels.yaml
with:
pr_number: ${{ inputs.pr_number }}
max_runtime: 20
@@ -25,10 +25,10 @@ jobs:
needs: get_Ready
steps:
- run: |
- if [[ ${{ needs.get_Ready.outputs.state }} == "false" ]]; then
+ if [[ "${{ needs.get_Ready.outputs.state }}" == "false" ]]; then
echo "Ready Timmed out"
exit 1
- elif [[ ${{ needs.get_Ready.outputs.state }} == "true" ]]; then
+ elif [[ "${{ needs.get_Ready.outputs.state }}" == "true" ]]; then
echo "Ready Set"
fi
--
GitLab
From f1767468407454c2cf6bc258eda4a00f49e48369 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Fri, 15 Dec 2023 15:10:53 -0500
Subject: [PATCH 039/189] no op for self test
---
ci/scripts/driver.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index a0edb4b4c..e2d41c2db 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -12,6 +12,7 @@ set -eux
# development branch for the global-workflow repo. It then stages tests directories per
# PR number and calls clone-build_ci.sh to perform a clone and full build from the PR.
# It then is ready to run a suite of regression tests with various configurations
+# no-up for self-test
#######################################################################################
#################################################################
--
GitLab
From 3cff605a0ddb072156ff65245b194456189f0d54 Mon Sep 17 00:00:00 2001
From: "terrance.mcguinness" <terrance.mcguinness@noaa.gov>
Date: Fri, 15 Dec 2023 15:13:34 -0500
Subject: [PATCH 040/189] no op for self test
---
ci/scripts/driver.sh | 1 -
1 file changed, 1 deletion(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index e2d41c2db..a0edb4b4c 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -12,7 +12,6 @@ set -eux
# development branch for the global-workflow repo. It then stages tests directories per
# PR number and calls clone-build_ci.sh to perform a clone and full build from the PR.
# It then is ready to run a suite of regression tests with various configurations
-# no-up for self-test
#######################################################################################
#################################################################
--
GitLab
From cef725054194d3c7518a8a4a573c4a6e578b2610 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 11 Mar 2024 22:01:13 +0000
Subject: [PATCH 041/189] adde pygithug into pr_list_database
---
ci/scripts/githubpr.py | 74 ++++++++++++++++++++++++++++++
ci/scripts/pr_list_database.py | 82 ++++++++++++++++++++++------------
ci/scripts/workflow | 1 +
3 files changed, 128 insertions(+), 29 deletions(-)
create mode 100755 ci/scripts/githubpr.py
create mode 120000 ci/scripts/workflow
diff --git a/ci/scripts/githubpr.py b/ci/scripts/githubpr.py
new file mode 100755
index 000000000..d96df46ab
--- /dev/null
+++ b/ci/scripts/githubpr.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import os, sys
+import re
+
+from github import Github
+from wxflow import which
+from workflow.hosts import Host
+
+class GitHubPR:
+ """
+ GitHubPR A class for interacting with GitHub pull requests.
+
+ Attributes
+ ----------
+ gh : Github
+ The Github instance for interacting with the GitHub API.
+ repo : Repository
+ The GitHub repository.
+ host : Host
+ The host machine.
+ """
+
+ def __init__(self):
+ """
+ __init__ Initialize a new GitHubPR instance.
+
+ This method authenticates with the GitHub API using the 'gh' CLI tool,
+ gets the repository from the 'REPO_URL' environment variable, and
+ initializes the host machine.
+ """
+ gh_cli = which('gh')
+ gh_cli.add_default_arg(['auth', 'status','--show-token'])
+ gh_access_token=gh_cli(output=str, error=str).split('\n')[3].split(': ')[1]
+ gh = Github(gh_access_token)
+ self.gh = gh
+
+ repo_url = os.environ.get("REPO_URL")
+ match = re.search(r"github\.com/(.+)", repo_url)
+ repo_identifier = match.group(1)[:-4]
+ repo = gh.get_repo(repo_identifier)
+ self.repo = repo
+ self.host = Host()
+
+ def get_open_pr_list(self,state='Ready'):
+ """
+ get_open_pr_list Get a list of open pull requests.
+
+ Parameters
+ ----------
+ state : str, optional
+ The state of the pull requests to get (default is 'Ready').
+
+ Returns
+ -------
+ list
+ A list of pull request numbers that are open and match the specified state.
+ list
+ A list of pull request numbers that have the 'Kill' label.
+ """
+ pulls = self.repo.get_pulls(state='closed', sort='updated', direction='desc')
+ pr_list = []
+ pr_kill_list = []
+ for pull in pulls:
+ labels = pull.get_labels()
+ ci_labels = [s for s in labels if 'CI' in s.name]
+ for label in ci_labels:
+ if 'Kill' in label.name:
+ pr_kill_list.append(pull.number)
+ continue
+ if self.host.machine.capitalize() in label.name:
+ if state in label.name:
+ pr_list.append(pull.number)
+ return pr_list, pr_kill_list
\ No newline at end of file
diff --git a/ci/scripts/pr_list_database.py b/ci/scripts/pr_list_database.py
index f525d6498..fc04bbe98 100755
--- a/ci/scripts/pr_list_database.py
+++ b/ci/scripts/pr_list_database.py
@@ -3,37 +3,20 @@
import sys
import os
from wxflow import SQLiteDB, SQLiteDBError
+from githubpr import GitHubPR
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER
def full_path(string):
- """
- full_path Get the absolute path of a file or directory.
-
- Parameters
- ----------
- string : str
- The relative path of the file or directory.
-
- Returns
- -------
- str
- The absolute path of the file or directory.
-
- Raises
- ------
- NotADirectoryError
- If the provided string does not represent a valid file or directory.
- """
if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)):
return os.path.abspath(string)
else:
raise NotADirectoryError(string)
-def create_table(db: SQLiteDB):
+def create(db: SQLiteDB):
"""
- Create a new table in a database.
+ Create a new database.
Parameters
----------
@@ -43,7 +26,7 @@ def create_table(db: SQLiteDB):
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])
-def add_pr(db: SQLiteDB, pr: str):
+def add_pr(db: SQLiteDB, pr: str) -> bool:
"""
Add a pull request to the database.
@@ -57,9 +40,11 @@ def add_pr(db: SQLiteDB, pr: str):
entities = (pr, 'Open', 'Ready', 0, 'ci_repo')
try:
db.insert_data('pr_list', entities)
+ return True
except (SQLiteDBError.IntegrityError) as e:
if 'unique' in str(e).lower():
print(f"pr {pr} already is in list: nothing added")
+ return False
def update_pr(db: SQLiteDB, args):
@@ -68,7 +53,7 @@ def update_pr(db: SQLiteDB, args):
Parameters
----------
- db : SQLiteDB
+ ci_database : SQLiteDB
The database to update the pull request in.
args : argparse.Namespace
The command line arguments.
@@ -83,7 +68,7 @@ def update_pr(db: SQLiteDB, args):
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display_db(db, display):
+def display(db, display) -> list:
"""
Display the database.
@@ -91,8 +76,8 @@ def display_db(db, display):
----------
ci_database : SQLiteDB
The database to display.
- display : list
- The command line argument values.
+ args : argparse.Namespace
+ The command line arguments.
Returns
-------
@@ -101,14 +86,42 @@ def display_db(db, display):
"""
values = []
if len(display) == 1:
- rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f'pr = {display[0]}')
- else:
+ rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
+ if len(display) == 2:
+ #rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"state = '{display[0]}' AND status = '{display[1]}'")
+ rows = db.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
+ if len(display) == 0:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
for row in rows:
values.append(' '.join(map(str, row)))
return values
+def update_database(db: SQLiteDB) -> list:
+ """
+ Update the database from the GitHub PRs
+ - only PRs from host machine are added to the database
+ - if the PR is already in the database it its added to the kill list
+
+ Parameters
+ ----------
+ ci_database : SQLiteDB
+ The database to update.
+
+ Returns
+ -------
+ list
+ The kill list of pull requests.
+ """
+ gh = GitHubPR()
+ pr_ready_list, pr_kill_list = gh.get_open_pr_list()
+ for pr in pr_ready_list:
+ if not add_pr(db, str(pr)):
+ if pr not in pr_kill_list:
+ pr_kill_list.append(pr)
+ pr_kill_list = list(set(pr_kill_list))
+ return pr_kill_list
+
def input_args():
"""
@@ -132,6 +145,8 @@ def input_args():
parser.add_argument('--update_pr', nargs=REMAINDER, metavar=('pr', 'state', 'status', 'reset_id', 'cases'),
help='updates state and status of a given pr', required=False)
parser.add_argument('--display', nargs='*', help='output pr table', required=False)
+ parser.add_argument('--list_open_ready', action='store_true', required=False)
+ parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list', action='store_true', required=False)
args = parser.parse_args()
return args
@@ -149,7 +164,7 @@ if __name__ == '__main__':
ci_database.connect()
if args.create:
- create_table(ci_database)
+ create(ci_database)
if args.add_pr:
add_pr(ci_database, args.add_pr[0])
if args.update_pr:
@@ -157,7 +172,16 @@ if __name__ == '__main__':
if args.remove_pr:
ci_database.remove_data('pr_list', 'PR', args.remove_pr[0])
if args.display is not None:
- for rows in display_db(ci_database, args.display):
+ for rows in display(ci_database, args.display):
print(rows)
+ if args.list_open_ready:
+ for rows in display(ci_database, ['Open', 'Ready']):
+ print(rows, end=' ')
+ print()
+ if args.update_database:
+ pr_kill_list = update_database(ci_database)
+ for pr in pr_kill_list:
+ print(pr, end=' ')
+ print()
ci_database.disconnect()
diff --git a/ci/scripts/workflow b/ci/scripts/workflow
new file mode 120000
index 000000000..b3927c5ad
--- /dev/null
+++ b/ci/scripts/workflow
@@ -0,0 +1 @@
+../../workflow
\ No newline at end of file
--
GitLab
From 44f3f46e8fe8e3fd48a918a3d5475370e379b095 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Tue, 12 Mar 2024 12:13:09 -0500
Subject: [PATCH 042/189] remove extra gitworkfow work stuck in my forked
develop branch
---
.github/workflows/getlabels.yaml | 60 -----------------------
.github/workflows/orion_status.yaml | 75 -----------------------------
2 files changed, 135 deletions(-)
delete mode 100644 .github/workflows/getlabels.yaml
delete mode 100644 .github/workflows/orion_status.yaml
diff --git a/.github/workflows/getlabels.yaml b/.github/workflows/getlabels.yaml
deleted file mode 100644
index cff128217..000000000
--- a/.github/workflows/getlabels.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-name: Get Labels
-
-on:
- workflow_call:
- outputs:
- state:
- description: "The return state of Stage (true/false)"
- value: ${{ jobs.getlabels.outputs.state }}
- inputs:
- pr_number:
- required: true
- type: string
- max_runtime:
- required: false
- type: string
- stage:
- required: true
- type: string
-permissions:
- contents: read
- packages: write
-
-jobs:
-
- getlabels:
- runs-on: ubuntu-latest
- outputs:
- state: ${{ steps.id.outputs.state }}
- steps:
- - name: Get Label Steps
- id: id
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- OWNER: ${{ github.repository_owner }}
- REPO_NAME: ${{ github.event.repository.name }}
- PULL_REQUEST_NUMBER: ${{ inputs.pr_number }}
- STAGE: ${{ inputs.stage }}
- MAX_TIME: ${{ inputs.max_runtime }}
- run: |
- DONE=false
- count=0
- until false
- do
- echo "gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name'"
- LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')"
- LABELS=$(echo "$LABELS1" | tr '\n' ' ')
- check_label="CI-Orion-${STAGE}"
- if [[ "${LABELS}" == *"${check_label}"* ]]; then
- DONE=true
- break
- fi
- sleep 10m
- count=$((count+10))
- if [[ ${count} -gt ${MAX_TIME} ]]; then
- DONE=false
- break
- fi
- done
- echo "state=${DONE}"
- echo "state=${DONE}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/orion_status.yaml b/.github/workflows/orion_status.yaml
deleted file mode 100644
index 76d557127..000000000
--- a/.github/workflows/orion_status.yaml
+++ /dev/null
@@ -1,75 +0,0 @@
-name: OrionStatus
-
-on:
- workflow_dispatch:
- inputs:
- pr_number:
- description: PR number
- type: string
- max_runtime:
- description: Maximum time out time for running experment
- type: string
-
-jobs:
-
- get_Ready:
- uses: ./.github/workflows/getlabels.yaml
- with:
- pr_number: ${{ inputs.pr_number }}
- max_runtime: 20
- stage: "Ready"
- secrets: inherit
-
- Ready:
- runs-on: ubuntu-latest
- needs: get_Ready
- steps:
- - run: |
- if [[ "${{ needs.get_Ready.outputs.state }}" == "false" ]]; then
- echo "Ready Timmed out"
- exit 1
- elif [[ "${{ needs.get_Ready.outputs.state }}" == "true" ]]; then
- echo "Ready Set"
- fi
-
- get_Building:
- uses: ./.github/workflows/getlabels.yaml
- needs: Ready
- with:
- pr_number: ${{ inputs.pr_number }}
- max_runtime: 40
- stage: "Building"
- secrets: inherit
-
- Building:
- runs-on: ubuntu-latest
- needs: get_Building
- steps:
- - run: |
- if [[ "${{ needs.get_Building.outputs.state }}" == "false" ]]; then
- echo "Building Timmed out"
- exit 1
- elif [[ "${{ needs.get_Building.outputs.state }}" == "true" ]]; then
- echo "Building Set"
- fi
-
- get_Running:
- uses: ./.github/workflows/getlabels.yaml
- needs: Building
- with:
- pr_number: ${{ inputs.pr_number }}
- max_runtime: 60
- stage: "Running"
- secrets: inherit
-
- Running:
- runs-on: ubuntu-latest
- needs: get_Running
- steps:
- - run: |
- if [[ "${{ needs.get_Running.outputs.state }}" == "false" ]]; then
- echo "Running Timmed out"
- exit 1
- elif [[ "${{ needs.get_Running.outputs.state }}" == "true" ]]; then
- echo "Running Set"
- fi
\ No newline at end of file
--
GitLab
From 38ff559ce931ea1063347f8d1ef7215716c14dd3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 22 Mar 2024 14:46:12 +0000
Subject: [PATCH 043/189] updating devlop to match authoritative repo
---
ci/scripts/githubpr.py | 74 ------------------------------
ci/scripts/pr_list_database.py | 82 ++++++++++++----------------------
ci/scripts/workflow | 1 -
3 files changed, 29 insertions(+), 128 deletions(-)
delete mode 100755 ci/scripts/githubpr.py
delete mode 120000 ci/scripts/workflow
diff --git a/ci/scripts/githubpr.py b/ci/scripts/githubpr.py
deleted file mode 100755
index d96df46ab..000000000
--- a/ci/scripts/githubpr.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-import os, sys
-import re
-
-from github import Github
-from wxflow import which
-from workflow.hosts import Host
-
-class GitHubPR:
- """
- GitHubPR A class for interacting with GitHub pull requests.
-
- Attributes
- ----------
- gh : Github
- The Github instance for interacting with the GitHub API.
- repo : Repository
- The GitHub repository.
- host : Host
- The host machine.
- """
-
- def __init__(self):
- """
- __init__ Initialize a new GitHubPR instance.
-
- This method authenticates with the GitHub API using the 'gh' CLI tool,
- gets the repository from the 'REPO_URL' environment variable, and
- initializes the host machine.
- """
- gh_cli = which('gh')
- gh_cli.add_default_arg(['auth', 'status','--show-token'])
- gh_access_token=gh_cli(output=str, error=str).split('\n')[3].split(': ')[1]
- gh = Github(gh_access_token)
- self.gh = gh
-
- repo_url = os.environ.get("REPO_URL")
- match = re.search(r"github\.com/(.+)", repo_url)
- repo_identifier = match.group(1)[:-4]
- repo = gh.get_repo(repo_identifier)
- self.repo = repo
- self.host = Host()
-
- def get_open_pr_list(self,state='Ready'):
- """
- get_open_pr_list Get a list of open pull requests.
-
- Parameters
- ----------
- state : str, optional
- The state of the pull requests to get (default is 'Ready').
-
- Returns
- -------
- list
- A list of pull request numbers that are open and match the specified state.
- list
- A list of pull request numbers that have the 'Kill' label.
- """
- pulls = self.repo.get_pulls(state='closed', sort='updated', direction='desc')
- pr_list = []
- pr_kill_list = []
- for pull in pulls:
- labels = pull.get_labels()
- ci_labels = [s for s in labels if 'CI' in s.name]
- for label in ci_labels:
- if 'Kill' in label.name:
- pr_kill_list.append(pull.number)
- continue
- if self.host.machine.capitalize() in label.name:
- if state in label.name:
- pr_list.append(pull.number)
- return pr_list, pr_kill_list
\ No newline at end of file
diff --git a/ci/scripts/pr_list_database.py b/ci/scripts/pr_list_database.py
index fc04bbe98..f525d6498 100755
--- a/ci/scripts/pr_list_database.py
+++ b/ci/scripts/pr_list_database.py
@@ -3,20 +3,37 @@
import sys
import os
from wxflow import SQLiteDB, SQLiteDBError
-from githubpr import GitHubPR
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER
def full_path(string):
+ """
+ full_path Get the absolute path of a file or directory.
+
+ Parameters
+ ----------
+ string : str
+ The relative path of the file or directory.
+
+ Returns
+ -------
+ str
+ The absolute path of the file or directory.
+
+ Raises
+ ------
+ NotADirectoryError
+ If the provided string does not represent a valid file or directory.
+ """
if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)):
return os.path.abspath(string)
else:
raise NotADirectoryError(string)
-def create(db: SQLiteDB):
+def create_table(db: SQLiteDB):
"""
- Create a new database.
+ Create a new table in a database.
Parameters
----------
@@ -26,7 +43,7 @@ def create(db: SQLiteDB):
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])
-def add_pr(db: SQLiteDB, pr: str) -> bool:
+def add_pr(db: SQLiteDB, pr: str):
"""
Add a pull request to the database.
@@ -40,11 +57,9 @@ def add_pr(db: SQLiteDB, pr: str) -> bool:
entities = (pr, 'Open', 'Ready', 0, 'ci_repo')
try:
db.insert_data('pr_list', entities)
- return True
except (SQLiteDBError.IntegrityError) as e:
if 'unique' in str(e).lower():
print(f"pr {pr} already is in list: nothing added")
- return False
def update_pr(db: SQLiteDB, args):
@@ -53,7 +68,7 @@ def update_pr(db: SQLiteDB, args):
Parameters
----------
- ci_database : SQLiteDB
+ db : SQLiteDB
The database to update the pull request in.
args : argparse.Namespace
The command line arguments.
@@ -68,7 +83,7 @@ def update_pr(db: SQLiteDB, args):
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display(db, display) -> list:
+def display_db(db, display):
"""
Display the database.
@@ -76,8 +91,8 @@ def display(db, display) -> list:
----------
ci_database : SQLiteDB
The database to display.
- args : argparse.Namespace
- The command line arguments.
+ display : list
+ The command line argument values.
Returns
-------
@@ -86,42 +101,14 @@ def display(db, display) -> list:
"""
values = []
if len(display) == 1:
- rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
- if len(display) == 2:
- #rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"state = '{display[0]}' AND status = '{display[1]}'")
- rows = db.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
- if len(display) == 0:
+ rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f'pr = {display[0]}')
+ else:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
for row in rows:
values.append(' '.join(map(str, row)))
return values
-def update_database(db: SQLiteDB) -> list:
- """
- Update the database from the GitHub PRs
- - only PRs from host machine are added to the database
- - if the PR is already in the database it its added to the kill list
-
- Parameters
- ----------
- ci_database : SQLiteDB
- The database to update.
-
- Returns
- -------
- list
- The kill list of pull requests.
- """
- gh = GitHubPR()
- pr_ready_list, pr_kill_list = gh.get_open_pr_list()
- for pr in pr_ready_list:
- if not add_pr(db, str(pr)):
- if pr not in pr_kill_list:
- pr_kill_list.append(pr)
- pr_kill_list = list(set(pr_kill_list))
- return pr_kill_list
-
def input_args():
"""
@@ -145,8 +132,6 @@ def input_args():
parser.add_argument('--update_pr', nargs=REMAINDER, metavar=('pr', 'state', 'status', 'reset_id', 'cases'),
help='updates state and status of a given pr', required=False)
parser.add_argument('--display', nargs='*', help='output pr table', required=False)
- parser.add_argument('--list_open_ready', action='store_true', required=False)
- parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list', action='store_true', required=False)
args = parser.parse_args()
return args
@@ -164,7 +149,7 @@ if __name__ == '__main__':
ci_database.connect()
if args.create:
- create(ci_database)
+ create_table(ci_database)
if args.add_pr:
add_pr(ci_database, args.add_pr[0])
if args.update_pr:
@@ -172,16 +157,7 @@ if __name__ == '__main__':
if args.remove_pr:
ci_database.remove_data('pr_list', 'PR', args.remove_pr[0])
if args.display is not None:
- for rows in display(ci_database, args.display):
+ for rows in display_db(ci_database, args.display):
print(rows)
- if args.list_open_ready:
- for rows in display(ci_database, ['Open', 'Ready']):
- print(rows, end=' ')
- print()
- if args.update_database:
- pr_kill_list = update_database(ci_database)
- for pr in pr_kill_list:
- print(pr, end=' ')
- print()
ci_database.disconnect()
diff --git a/ci/scripts/workflow b/ci/scripts/workflow
deleted file mode 120000
index b3927c5ad..000000000
--- a/ci/scripts/workflow
+++ /dev/null
@@ -1 +0,0 @@
-../../workflow
\ No newline at end of file
--
GitLab
From c130382e23361bb20b735a42951f2dbdbb1c2527 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 17:29:51 +0000
Subject: [PATCH 044/189] added new feature of pr_list_database for getting pr
list and added rocoto_statcount for checkout case states that now also
includes checks for stalled
---
ci/scripts/check_ci.sh | 41 ++++----
ci/scripts/driver.sh | 26 ++---
ci/scripts/run_ci.sh | 3 +-
ci/scripts/{ => utils}/pr_list_database.py | 82 ++++++++++------
ci/scripts/utils/rocoto_statcount.py | 105 +++++++++++++++++++++
5 files changed, 197 insertions(+), 60 deletions(-)
rename ci/scripts/{ => utils}/pr_list_database.py (67%)
create mode 100755 ci/scripts/utils/rocoto_statcount.py
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 4ff7eefd2..5236fceb1 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -58,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
pr_list=""
if [[ -f "${pr_list_dbfile}" ]]; then
- pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+ pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
fi
if [[ -z "${pr_list+x}" ]]; then
echo "no PRs open and ready to run cases on .. exiting"
@@ -123,21 +123,27 @@ for pr in ${pr_list}; do
if [[ ! -f "${db}" ]]; then
continue
fi
- rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true
- num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true
- num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true
- # num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
- echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true
- num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true
- if [[ ${num_failed} -ne 0 ]]; then
+
+ set +e
+ rocoto_stat=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}") || true
+ set -e
+ rocoto_error=$?
+ if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
- error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- {
- echo "Experiment ${pslot} *** FAILED *** on ${MACHINE_ID^}"
- echo "Experiment ${pslot} with ${num_failed} tasks failed at $(date +'%D %r')" || true
- echo "Error logs:"
- echo "${error_logs}"
- } >> "${output_ci}"
+ # Check if the experiment failed due to a missing dependency and is stalled
+ if [[ "${rocoto_error}" -eq -3 ]]; then
+ echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}" >> "${output_ci_single}"
+ echo "Experiment ${pslot} with ${rocoto_error} at $(date +'%D %r') on ${MACHINE_ID^}" >> "${output_ci}"
+ # TODO used rocotocheck to find the missing dependency
+ else
+ error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
+ {
+ echo "Experiment ${pslot} *** ${rocoto_state} *** on ${MACHINE_ID^}"
+ echo "Experiment ${pslot} with ${rocoto_error} tasks failed at $(date +'%D %r')" || true
+ echo "Error logs:"
+ echo "${error_logs}"
+ } >> "${output_ci}"
+ fi
sed -i "1 i\`\`\`" "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
"${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
@@ -147,7 +153,9 @@ for pr in ${pr_list}; do
done
break
fi
- if [[ "${num_done}" -eq "${num_cycles}" ]]; then
+ echo ${rocoto_stat}
+ rocoto_state=$(echo -e "${rocoto_stat}" | tail -1)
+ if [[ "${rocoto_state}" -eq "DONE" ]]; then
#Remove Experment cases that completed successfully
rm -Rf "${pslot_dir}"
rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
@@ -157,7 +165,6 @@ for pr in ${pr_list}; do
echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}"
echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
-
fi
done
done
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index f37b5e3f2..8a71bb4d4 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -64,14 +64,14 @@ unset HOMEgfs
pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
if [[ ! -f "${pr_list_dbfile}" ]]; then
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --create --dbfile "${pr_list_dbfile}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}"
fi
pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true
for pr in ${pr_list}; do
pr_dir="${GFS_CI_ROOT}/PR/${pr}"
- db_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
+ db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log"
#############################################################
# Check if a Ready labeled PR has changed back from once set
@@ -82,7 +82,7 @@ for pr in ${pr_list}; do
if [[ "${db_list}" == *"already is in list"* ]]; then
# Get the the PID and HOST of the driver.sh cron job
# that is stored int he CI database for this PR
- driver_ID=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display "${pr}" | awk '{print $4}') || true
+ driver_ID=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display "${pr}" | awk '{print $4}') || true
driver_PID=$(echo "${driver_ID}" | cut -d":" -f1) || true
driver_HOST=$(echo "${driver_ID}" | cut -d":" -f2) || true
host_name=$(hostname -s)
@@ -121,14 +121,14 @@ for pr in ${pr_list}; do
fi
sed -i "1 i\`\`\`" "${output_ci_single}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}"
fi
done
pr_list=""
if [[ -f "${pr_list_dbfile}" ]]; then
- pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Ready | awk '{print $1}') || true
+ pr_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true
fi
if [[ -z "${pr_list+x}" ]]; then
echo "no PRs open and ready for checkout/build .. exiting"
@@ -143,7 +143,7 @@ fi
for pr in ${pr_list}; do
# Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron
- pr_building=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true
+ pr_building=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true
if [[ -z "${pr_building+x}" ]]; then
continue
fi
@@ -154,7 +154,7 @@ for pr in ${pr_list}; do
driver_build_PID=$$
driver_build_HOST=$(hostname -s)
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building "${driver_build_PID}:${driver_build_HOST}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building "${driver_build_PID}:${driver_build_HOST}"
rm -Rf "${pr_dir}"
mkdir -p "${pr_dir}"
{
@@ -176,7 +176,7 @@ for pr in ${pr_list}; do
# we need to exit this instance of the driver script
#################################################################
if [[ ${ci_status} -ne 0 ]]; then
- build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
+ build_PID_check=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
if [[ "${build_PID_check}" -ne "$$" ]]; then
echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting"
exit 0
@@ -184,7 +184,7 @@ for pr in ${pr_list}; do
fi
set -e
if [[ ${ci_status} -eq 0 ]]; then
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built "0:0"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built "0:0"
#setup space to put an experiment
# export RUNTESTS for yaml case files to pickup
export RUNTESTS="${pr_dir}/RUNTESTS"
@@ -226,14 +226,14 @@ for pr in ${pr_list}; do
cat "${LOGFILE_PATH}"
} >> "${output_ci}"
"${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
exit 1
fi
done
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
else
@@ -242,7 +242,7 @@ for pr in ${pr_list}; do
echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
} >> "${output_ci}"
"${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
- "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
fi
diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
index f50a4465d..8af35bc47 100755
--- a/ci/scripts/run_ci.sh
+++ b/ci/scripts/run_ci.sh
@@ -48,7 +48,8 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
pr_list=""
if [[ -f "${pr_list_dbfile}" ]]; then
- pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
+ pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
+ pr_list=$(echo "${pr_list}" | tr ' ' '\n' | head -n "${max_concurrent_pr}" | tr '\n' ' ') || true
fi
if [[ -z "${pr_list}" ]]; then
echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"
diff --git a/ci/scripts/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
similarity index 67%
rename from ci/scripts/pr_list_database.py
rename to ci/scripts/utils/pr_list_database.py
index f525d6498..dc7531498 100755
--- a/ci/scripts/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -3,37 +3,20 @@
import sys
import os
from wxflow import SQLiteDB, SQLiteDBError
+from githubpr import GitHubPR
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER
def full_path(string):
- """
- full_path Get the absolute path of a file or directory.
-
- Parameters
- ----------
- string : str
- The relative path of the file or directory.
-
- Returns
- -------
- str
- The absolute path of the file or directory.
-
- Raises
- ------
- NotADirectoryError
- If the provided string does not represent a valid file or directory.
- """
if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)):
return os.path.abspath(string)
else:
raise NotADirectoryError(string)
-def create_table(db: SQLiteDB):
+def create(db: SQLiteDB):
"""
- Create a new table in a database.
+ Create a new database.
Parameters
----------
@@ -43,7 +26,7 @@ def create_table(db: SQLiteDB):
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])
-def add_pr(db: SQLiteDB, pr: str):
+def add_pr(db: SQLiteDB, pr: str) -> bool:
"""
Add a pull request to the database.
@@ -57,9 +40,11 @@ def add_pr(db: SQLiteDB, pr: str):
entities = (pr, 'Open', 'Ready', 0, 'ci_repo')
try:
db.insert_data('pr_list', entities)
+ return True
except (SQLiteDBError.IntegrityError) as e:
if 'unique' in str(e).lower():
print(f"pr {pr} already is in list: nothing added")
+ return False
def update_pr(db: SQLiteDB, args):
@@ -68,7 +53,7 @@ def update_pr(db: SQLiteDB, args):
Parameters
----------
- db : SQLiteDB
+ ci_database : SQLiteDB
The database to update the pull request in.
args : argparse.Namespace
The command line arguments.
@@ -83,7 +68,7 @@ def update_pr(db: SQLiteDB, args):
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display_db(db, display):
+def display(db, display) -> list:
"""
Display the database.
@@ -91,8 +76,8 @@ def display_db(db, display):
----------
ci_database : SQLiteDB
The database to display.
- display : list
- The command line argument values.
+ args : argparse.Namespace
+ The command line arguments.
Returns
-------
@@ -101,14 +86,42 @@ def display_db(db, display):
"""
values = []
if len(display) == 1:
- rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f'pr = {display[0]}')
- else:
+ rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
+ if len(display) == 2:
+ #rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"state = '{display[0]}' AND status = '{display[1]}'")
+ rows = db.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
+ if len(display) == 0:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
for row in rows:
values.append(' '.join(map(str, row)))
return values
+def update_database(db: SQLiteDB) -> list:
+ """
+ Update the database from the GitHub PRs
+ - only PRs from host machine are added to the database
+ - if the PR is already in the database it its added to the kill list
+
+ Parameters
+ ----------
+ ci_database : SQLiteDB
+ The database to update.
+
+ Returns
+ -------
+ list
+ The kill list of pull requests.
+ """
+ gh = GitHubPR()
+ pr_ready_list, pr_kill_list = gh.get_open_pr_list()
+ for pr in pr_ready_list:
+ if not add_pr(db, str(pr)):
+ if pr not in pr_kill_list:
+ pr_kill_list.append(pr)
+ pr_kill_list = list(set(pr_kill_list))
+ return pr_kill_list
+
def input_args():
"""
@@ -132,6 +145,8 @@ def input_args():
parser.add_argument('--update_pr', nargs=REMAINDER, metavar=('pr', 'state', 'status', 'reset_id', 'cases'),
help='updates state and status of a given pr', required=False)
parser.add_argument('--display', nargs='*', help='output pr table', required=False)
+ parser.add_argument('--list', nargs=2, metavar=('state', 'status'), required=False)
+ parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list', action='store_true', required=False)
args = parser.parse_args()
return args
@@ -149,7 +164,7 @@ if __name__ == '__main__':
ci_database.connect()
if args.create:
- create_table(ci_database)
+ create(ci_database)
if args.add_pr:
add_pr(ci_database, args.add_pr[0])
if args.update_pr:
@@ -157,7 +172,16 @@ if __name__ == '__main__':
if args.remove_pr:
ci_database.remove_data('pr_list', 'PR', args.remove_pr[0])
if args.display is not None:
- for rows in display_db(ci_database, args.display):
+ for rows in display(ci_database, args.display):
print(rows)
+ if args.list:
+ for rows in display(ci_database, [args.list[0], args.list[1]]):
+ print(rows, end=' ')
+ print()
+ if args.update_database:
+ pr_kill_list = update_database(ci_database)
+ for pr in pr_kill_list:
+ print(pr, end=' ')
+ print()
ci_database.disconnect()
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
new file mode 100755
index 000000000..bd623fd12
--- /dev/null
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+
+from wxflow import Executable, which, Logger, CommandNotFoundError
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
+
+
+def input_args():
+ """
+ Parse command-line arguments.
+
+ Returns
+ -------
+ args : Namespace
+ The parsed command-line arguments.
+ """
+
+ description = """
+ Using rocotostat to get the status of all jobs this scripts
+ determines rocoto_state: if all cycles are done, then rocoto_state is Done.
+ Assuming rocotorun had just been run, and the rocoto_state is not Done, then
+ rocoto_state is Stalled if there are no jobs that are RUNNING, SUBMITTING, or QUEUED.
+ """
+
+ parser = ArgumentParser(description=description,
+ formatter_class=ArgumentDefaultsHelpFormatter)
+
+ parser.add_argument('-w', help='workflow_document', type=str)
+ parser.add_argument('-d', help='database_file', type=str)
+
+ args = parser.parse_args()
+
+ return args
+
+
+def rocoto_statcount():
+ """
+ Run rocotostat and process its output.
+ """
+
+ args = input_args()
+
+ try:
+ rocotostat = which("rocotostat")
+ except CommandNotFoundError:
+ logger.exception("rocotostat not found in PATH")
+ raise CommandNotFoundError("rocotostat not found in PATH")
+
+ xml_file_path = os.path.abspath(args.w)
+ db_file_path = os.path.abspath(args.d)
+
+ rocotostat_all = which("rocotostat")
+ rocotostat.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-s'])
+ rocotostat_all.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-a'])
+
+ rocotostat_output = rocotostat(output=str)
+ rocotostat_output = rocotostat_output.splitlines()[1:]
+ rocotostat_output = [line.split()[0:2] for line in rocotostat_output]
+
+ rocotostat_output_all = rocotostat_all(output=str)
+ rocotostat_output_all = rocotostat_output_all.splitlines()[1:]
+ rocotostat_output_all = [line.split()[0:4] for line in rocotostat_output_all]
+ rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1]
+
+ rocoto_status = {
+ 'Cycles': len(rocotostat_output),
+ 'Cycles_Done': sum([sublist.count('Done') for sublist in rocotostat_output])
+ }
+
+ status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED']
+ for case in status_cases:
+ rocoto_status[case] = sum([sublist.count(case) for sublist in rocotostat_output_all])
+
+ return rocoto_status
+
+
+if __name__ == '__main__':
+
+ args = input_args()
+
+ error_return = 0
+ rocoto_status = rocoto_statcount()
+ if rocoto_status['Cycles'] == rocoto_status['Done_Cycles']:
+ print(f'All {rocoto_status['Cycles']} Cycles are Done')
+ rocoto_state = 'DONE'
+ elif rocoto_status['DEAD'] > 0:
+ error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
+ rocoto_state = 'FAILED or DEAD'
+ elif 'UNKNOWN' in rocoto_status:
+ error_return = rocoto_status['UNKNOWN']
+ rocoto_state = 'UNKNOWN'
+ elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0:
+ error_return = -3
+ rocoto_state = 'STALLED'
+ else:
+ for status in rocoto_status:
+ print(f'Number of {status} : {rocoto_status[status]}')
+ rocoto_state = 'RUNNING'
+
+ print(rocoto_state)
+ sys.exit(error_return)
--
GitLab
From 8f346dafacb90a921990b746d0ecdd6a17c11aa3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:30:45 +0000
Subject: [PATCH 045/189] missed some updates from previouse review on pr list
database script
---
ci/scripts/utils/pr_list_database.py | 30 +++++++++++++++++++++-------
1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index dc7531498..f9833cf3f 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -8,15 +8,31 @@ from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER
def full_path(string):
+ """
+ full_path Get the absolute path of a file or directory.
+ Parameters
+ ----------
+ string : str
+ The relative path of the file or directory.
+ Returns
+ -------
+ str
+ The absolute path of the file or directory.
+ Raises
+ ------
+ NotADirectoryError
+ If the provided string does not represent a valid file or directory.
+ """
+
if os.path.isfile(string) or os.path.isdir(os.path.dirname(string)):
return os.path.abspath(string)
else:
raise NotADirectoryError(string)
-def create(db: SQLiteDB):
+def create_table(db: SQLiteDB):
"""
- Create a new database.
+ Create a new table in a database.
Parameters
----------
@@ -68,7 +84,7 @@ def update_pr(db: SQLiteDB, args):
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display(db, display) -> list:
+def display_db(db, display) -> list:
"""
Display the database.
@@ -155,7 +171,7 @@ if __name__ == '__main__':
args = input_args()
- if not args.create:
+ if not args.create_table:
if not os.path.isfile(args.dbfile):
print(f'Error: {args.dbfile} does not exsist')
sys.exit(-1)
@@ -164,7 +180,7 @@ if __name__ == '__main__':
ci_database.connect()
if args.create:
- create(ci_database)
+ create_table(ci_database)
if args.add_pr:
add_pr(ci_database, args.add_pr[0])
if args.update_pr:
@@ -172,10 +188,10 @@ if __name__ == '__main__':
if args.remove_pr:
ci_database.remove_data('pr_list', 'PR', args.remove_pr[0])
if args.display is not None:
- for rows in display(ci_database, args.display):
+ for rows in display_db(ci_database, args.display):
print(rows)
if args.list:
- for rows in display(ci_database, [args.list[0], args.list[1]]):
+ for rows in display_db(ci_database, [args.list[0], args.list[1]]):
print(rows, end=' ')
print()
if args.update_database:
--
GitLab
From 958994532081941cf842cedd2970a90d1dba279f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:38:22 +0000
Subject: [PATCH 046/189] a few pynorms spaces
---
ci/scripts/utils/pr_list_database.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index f9833cf3f..5667cf707 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -104,7 +104,6 @@ def display_db(db, display) -> list:
if len(display) == 1:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
if len(display) == 2:
- #rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"state = '{display[0]}' AND status = '{display[1]}'")
rows = db.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
if len(display) == 0:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
@@ -113,6 +112,7 @@ def display_db(db, display) -> list:
return values
+
def update_database(db: SQLiteDB) -> list:
"""
Update the database from the GitHub PRs
@@ -162,7 +162,8 @@ def input_args():
help='updates state and status of a given pr', required=False)
parser.add_argument('--display', nargs='*', help='output pr table', required=False)
parser.add_argument('--list', nargs=2, metavar=('state', 'status'), required=False)
- parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list', action='store_true', required=False)
+ parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list',
+ action='store_true', required=False)
args = parser.parse_args()
return args
--
GitLab
From 6e5e9980d71cb8bdc9143b4054952811d003d7c7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:43:56 +0000
Subject: [PATCH 047/189] extra spaces
---
ci/scripts/utils/rocoto_statcount.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index bd623fd12..01cf8705e 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -81,9 +81,10 @@ def rocoto_statcount():
if __name__ == '__main__':
args = input_args()
-
+
error_return = 0
rocoto_status = rocoto_statcount()
+
if rocoto_status['Cycles'] == rocoto_status['Done_Cycles']:
print(f'All {rocoto_status['Cycles']} Cycles are Done')
rocoto_state = 'DONE'
--
GitLab
From 0f1f8fd458ca4699aae596091267e9661f52cee9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:47:20 +0000
Subject: [PATCH 048/189] shell norms and spaces
---
ci/scripts/check_ci.sh | 2 +-
ci/scripts/utils/pr_list_database.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 5236fceb1..7632363ef 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -153,7 +153,7 @@ for pr in ${pr_list}; do
done
break
fi
- echo ${rocoto_stat}
+ echo "${rocoto_stat}"
rocoto_state=$(echo -e "${rocoto_stat}" | tail -1)
if [[ "${rocoto_state}" -eq "DONE" ]]; then
#Remove Experment cases that completed successfully
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index 5667cf707..ea2d91344 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -163,7 +163,7 @@ def input_args():
parser.add_argument('--display', nargs='*', help='output pr table', required=False)
parser.add_argument('--list', nargs=2, metavar=('state', 'status'), required=False)
parser.add_argument('--update_database', help='use labels from Open GitHub PRs to update database state and produces a kill list',
- action='store_true', required=False)
+ action='store_true', required=False)
args = parser.parse_args()
return args
--
GitLab
From 1d42f812c1c96692a21a37c5253cf3d7e6708d2f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:50:38 +0000
Subject: [PATCH 049/189] shell norm using ==
---
ci/scripts/check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 7632363ef..adc1c6122 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -155,7 +155,7 @@ for pr in ${pr_list}; do
fi
echo "${rocoto_stat}"
rocoto_state=$(echo -e "${rocoto_stat}" | tail -1)
- if [[ "${rocoto_state}" -eq "DONE" ]]; then
+ if [[ "${rocoto_state}" == "DONE" ]]; then
#Remove Experment cases that completed successfully
rm -Rf "${pslot_dir}"
rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
--
GitLab
From 0d7cde9bfd39a955b54868564b3edc30933a9aed Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:54:08 +0000
Subject: [PATCH 050/189] fixed date sring
---
ci/scripts/check_ci.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index adc1c6122..a085cd3ec 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -132,8 +132,9 @@ for pr in ${pr_list}; do
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
# Check if the experiment failed due to a missing dependency and is stalled
if [[ "${rocoto_error}" -eq -3 ]]; then
+ date=$(date +'%D %r')
echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}" >> "${output_ci_single}"
- echo "Experiment ${pslot} with ${rocoto_error} at $(date +'%D %r') on ${MACHINE_ID^}" >> "${output_ci}"
+ echo "Experiment ${pslot} with ${rocoto_error} at ${date} on ${MACHINE_ID^}" >> "${output_ci}"
# TODO used rocotocheck to find the missing dependency
else
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
--
GitLab
From e0d22e5aea61f343c2d157b144fed3aa16f0bfd5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 21:07:47 +0000
Subject: [PATCH 051/189] create_table is not the argumnet
---
ci/scripts/utils/pr_list_database.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index ea2d91344..f6af7880a 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -172,7 +172,7 @@ if __name__ == '__main__':
args = input_args()
- if not args.create_table:
+ if not args.create:
if not os.path.isfile(args.dbfile):
print(f'Error: {args.dbfile} does not exsist')
sys.exit(-1)
--
GitLab
From 5d3353ee06f396483b0358fc287237862866f42d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 23:09:06 +0000
Subject: [PATCH 052/189] missed utils in path to pr_list_database script
---
ci/scripts/check_ci.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index a085cd3ec..dc089b9ab 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -58,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
pr_list=""
if [[ -f "${pr_list_dbfile}" ]]; then
- pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
+ pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
fi
if [[ -z "${pr_list+x}" ]]; then
echo "no PRs open and ready to run cases on .. exiting"
@@ -90,7 +90,7 @@ for pr in ${pr_list}; do
sed -i "1 i\`\`\`" "${output_ci}"
sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
- "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
# Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}" --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
if [[ -n "${weekly_labels}" ]]; then
@@ -147,7 +147,7 @@ for pr in ${pr_list}; do
fi
sed -i "1 i\`\`\`" "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
- "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
for kill_cases in "${pr_dir}/RUNTESTS/"*; do
pslot=$(basename "${kill_cases}")
cancel_slurm_jobs "${pslot}"
--
GitLab
From 343e3c7193c65069807de62df765a528bc754f67 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 18:12:38 -0500
Subject: [PATCH 053/189] fixed quote in dictionary for rocoto stat
---
ci/scripts/utils/rocoto_statcount.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index 01cf8705e..346117235 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -85,8 +85,8 @@ if __name__ == '__main__':
error_return = 0
rocoto_status = rocoto_statcount()
- if rocoto_status['Cycles'] == rocoto_status['Done_Cycles']:
- print(f'All {rocoto_status['Cycles']} Cycles are Done')
+ if rocoto_status['Cycles'] == rocoto_status['Cycles_Done']:
+ print(f"All {rocoto_status['Cycles']} Cycles are Done")
rocoto_state = 'DONE'
elif rocoto_status['DEAD'] > 0:
error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
--
GitLab
From 9da13730d229f43569ac0833ad3ac949a5c9f918 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Mon, 1 Apr 2024 19:22:21 -0500
Subject: [PATCH 054/189] Finished debugging check_ci with python rocoto
checker
---
ci/scripts/check_ci.sh | 27 ++++++++++++---------------
1 file changed, 12 insertions(+), 15 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index dc089b9ab..3f72ca818 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -125,37 +125,34 @@ for pr in ${pr_list}; do
fi
set +e
- rocoto_stat=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}") || true
- set -e
+ rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}")
rocoto_error=$?
+ rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
# Check if the experiment failed due to a missing dependency and is stalled
- if [[ "${rocoto_error}" -eq -3 ]]; then
+ if [[ "${rocoto_state}" == "STALLED" ]]; then
date=$(date +'%D %r')
- echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}" >> "${output_ci_single}"
- echo "Experiment ${pslot} with ${rocoto_error} at ${date} on ${MACHINE_ID^}" >> "${output_ci}"
+ echo "Experiment ${pslot} with **${rocoto_state}** at ${date} on ${MACHINE_ID^}" >> "${output_ci_single}"
+ "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
# TODO used rocotocheck to find the missing dependency
else
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
{
echo "Experiment ${pslot} *** ${rocoto_state} *** on ${MACHINE_ID^}"
- echo "Experiment ${pslot} with ${rocoto_error} tasks failed at $(date +'%D %r')" || true
+ echo "Experiment ${pslot} with ${rocoto_state} tasks failed at $(date +'%D %r')" || true
echo "Error logs:"
echo "${error_logs}"
} >> "${output_ci}"
+ sed -i "1 i\`\`\`" "${output_ci}"
+ "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
fi
- sed -i "1 i\`\`\`" "${output_ci}"
- "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
"${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
- for kill_cases in "${pr_dir}/RUNTESTS/"*; do
- pslot=$(basename "${kill_cases}")
- cancel_slurm_jobs "${pslot}"
- done
- break
+ cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
+ exit "${rocoto_error}"
fi
- echo "${rocoto_stat}"
- rocoto_state=$(echo -e "${rocoto_stat}" | tail -1)
+ echo "${rocoto_state}"
+ rocoto_state=$(echo -e "${rocoto_state}" | tail -1)
if [[ "${rocoto_state}" == "DONE" ]]; then
#Remove Experment cases that completed successfully
rm -Rf "${pslot_dir}"
--
GitLab
From b0879df9069a24e3649dd493e2313e7e200283d3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 2 Apr 2024 18:06:02 +0000
Subject: [PATCH 055/189] added the verbose switch in the python rocotostat
count script
---
ci/scripts/check_ci.sh | 2 +-
ci/scripts/utils/rocoto_statcount.py | 14 ++++++++------
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 3f72ca818..ec880e71a 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -125,7 +125,7 @@ for pr in ${pr_list}; do
fi
set +e
- rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}")
+ rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db} --verbose")
rocoto_error=$?
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index 346117235..91f8ca4da 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -26,11 +26,11 @@ def input_args():
rocoto_state is Stalled if there are no jobs that are RUNNING, SUBMITTING, or QUEUED.
"""
- parser = ArgumentParser(description=description,
- formatter_class=ArgumentDefaultsHelpFormatter)
+ parser = ArgumentParser(description=description)
- parser.add_argument('-w', help='workflow_document', type=str)
- parser.add_argument('-d', help='database_file', type=str)
+ parser.add_argument('-w', help='workflow_document', nargs=1, metavar='XML Workflow File', type=str, required=True)
+ parser.add_argument('-d', help='database_file', nargs=1, metavar='Database File', type=str, required=True)
+ parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
args = parser.parse_args()
@@ -98,9 +98,11 @@ if __name__ == '__main__':
error_return = -3
rocoto_state = 'STALLED'
else:
+ rocoto_state = 'RUNNING'
+
+ if args.verbose:
for status in rocoto_status:
- print(f'Number of {status} : {rocoto_status[status]}')
- rocoto_state = 'RUNNING'
+ print(f'Number of {status} : {rocoto_status[status]}\n')
print(rocoto_state)
sys.exit(error_return)
--
GitLab
From 443dd873faaf24484fbb19abdece8bc7b9a38c62 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Tue, 2 Apr 2024 14:25:53 -0500
Subject: [PATCH 056/189] fixed driver from failing if the build PID that is in
the database no loger exists
---
ci/scripts/driver.sh | 19 ++++++++++++++-----
workflow/rocoto_viewer.py | 2 +-
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 8a71bb4d4..735f0b82a 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -18,7 +18,7 @@ set -eux
# TODO using static build for GitHub CLI until fixed in HPC-Stack
#################################################################
export GH=${HOME}/bin/gh
-export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+export REPO_URL=git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git
################################################################
# Setup the reletive paths to scripts and PS4 for better logging
@@ -96,7 +96,10 @@ for pr in ${pr_list}; do
echo "Driver PID: ${driver_PID} no longer running this build having it killed"
if [[ "${driver_HOST}" == "${host_name}" ]]; then
# shellcheck disable=SC2312
- pstree -A -p "${driver_PID}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
+ pstree_out=$(pstree -A -p "${driver_PID}")
+ if [[ -n "${pstree_out}" ]]; then
+ echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
+ fi
else
# shellcheck disable=SC2312
ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
@@ -113,13 +116,16 @@ for pr in ${pr_list}; do
else
for case in ${experiments}; do
case_name=$(basename "${case}")
- cancel_slurm_jobs "${case_name}"
+ cancel_batch_jobs "${case_name}"
{
echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}"
} >> "${output_ci_single}"
done
fi
- sed -i "1 i\`\`\`" "${output_ci_single}"
+ first_line=$(head -n 1 "${output_ci_single}")
+ if [[ "$first_line" != '```' ]]; then
+ sed -i "1 i\`\`\`" "${output_ci_single}"
+ fi
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
"${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
"${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}"
@@ -164,7 +170,10 @@ for pr in ${pr_list}; do
echo "with PID: ${driver_build_PID} on host: ${driver_build_HOST}"
echo ""
} >> "${output_ci_single}"
- sed -i "1 i\`\`\`" "${output_ci_single}"
+ first_line=$(head -n 1 "${output_ci_single}")
+ if [[ "$first_line" != '```' ]]; then
+ sed -i "1 i\`\`\`" "${output_ci_single}"
+ fi
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
set +e
"${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}"
diff --git a/workflow/rocoto_viewer.py b/workflow/rocoto_viewer.py
index 95dd9e76d..78936c720 100755
--- a/workflow/rocoto_viewer.py
+++ b/workflow/rocoto_viewer.py
@@ -1360,7 +1360,7 @@ def main(screen):
screen.refresh()
curses.mousemask(1)
curses.noecho()
- for i in range(0, curses.COLORS):
+ for i in range(0, curses.COLORS-1):
curses.init_pair(i + 1, i, curses.COLOR_BLACK)
if i == 4:
curses.init_pair(i + 1, i, curses.COLOR_WHITE)
--
GitLab
From 5a6887f981c209feb49f4cc3d01c82092a6c5403 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 15:20:46 +0000
Subject: [PATCH 057/189] added a bash wrapper to publish_logs so that it can
take a file with incomplete paths and publish them all to gitHub
---
ci/scripts/utils/ci_utils.sh | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index ce2e03930..316b05928 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -126,3 +126,28 @@ function create_experiment () {
"${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
}
+
+function publish_logs() {
+# publish_logs function
+# This function takes a directory path and a list of files as arguments.
+# It calls the publish_logs.py script to publish the logs and returns its gist URL.
+# Usage: publish_logs <dir_path> <file1> <file2> ... <fileN>
+ local file="$1"
+ local dir_path="$2"
+
+ local full_paths=""
+ while IFS= read -r line; do
+ full_path="${dir_path}/${line}"
+ if [[ -f "$full_path" ]]; then
+ full_paths+="$full_path "
+ else
+ echo "File $full_path does not exist"
+ fi
+ done < "$file"
+
+ if [[ -n "$full_paths" ]]; then
+ python publish_logs.py --file $full_paths --repo > /dev/null
+ URL=$(python publish_logs.py --file $full_paths --gist)
+ fi
+ echo "$URL"
+}
--
GitLab
From 36210b5b74841a86f29426f71e373f239cb4fed5 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 11:21:14 -0500
Subject: [PATCH 058/189] add wrapper for publishing error logs from a file and
pre path
---
ci/scripts/utils/ci_utils.sh | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 316b05928..01122c1b3 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -131,23 +131,24 @@ function publish_logs() {
# publish_logs function
# This function takes a directory path and a list of files as arguments.
# It calls the publish_logs.py script to publish the logs and returns its gist URL.
-# Usage: publish_logs <dir_path> <file1> <file2> ... <fileN>
- local file="$1"
+# Usage: publish_logs <ID> <dir_path> <file1> <file2> ... <fileN>
+ local PR_header="$1"
local dir_path="$2"
+ local file="$3"
local full_paths=""
while IFS= read -r line; do
full_path="${dir_path}/${line}"
- if [[ -f "$full_path" ]]; then
- full_paths+="$full_path "
+ if [[ -f "${full_path}" ]]; then
+ full_paths+="${full_path} "
else
- echo "File $full_path does not exist"
+ echo "File "${full_path}" does not exist"
fi
done < "$file"
- if [[ -n "$full_paths" ]]; then
- python publish_logs.py --file $full_paths --repo > /dev/null
- URL=$(python publish_logs.py --file $full_paths --gist)
+ if [[ -n "${full_paths}" ]]; then
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo "${PR_header}" > /dev/null
+ URL=$(${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --gist "${PR_header}")
fi
- echo "$URL"
+ echo "${URL}"
}
--
GitLab
From 359c65ed91bace74ee0359b1566d832b03e0a758 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 16:58:52 +0000
Subject: [PATCH 059/189] used utils bash wrapper to publish list of error logs
from the build into the main bash driver
---
ci/scripts/driver.sh | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 735f0b82a..4982a35fe 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -245,14 +245,25 @@ for pr in ${pr_list}; do
"${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
- else
+ else # failed to clone and build
+
{
echo "Failed on cloning and building global-workflowi PR: ${pr}"
echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
} >> "${output_ci}"
+
"${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
"${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+
+ if [[ -f "${HOMEgfs}/sorc/logs/error.logs" ]]; then
+ gist_URL=$("${ROOT_DIR}/ci/scripts/utils/ci_utils_wrapper.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs")
+ {
+ echo -e "\nError logs from build"
+ echo "Gist URL: ${gist_URL}"
+ } >> "${output_ci}"
+ fi
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
+
fi
done # looping over each open and labeled PR
--
GitLab
From 96abb93ea541e1500c88db965f5ec0be169ecf90 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 12:03:29 -0500
Subject: [PATCH 060/189] bash norms quotes
---
ci/scripts/driver.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 4982a35fe..a50ba3620 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -123,7 +123,7 @@ for pr in ${pr_list}; do
done
fi
first_line=$(head -n 1 "${output_ci_single}")
- if [[ "$first_line" != '```' ]]; then
+ if [[ "${first_line}" != '```' ]]; then
sed -i "1 i\`\`\`" "${output_ci_single}"
fi
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
@@ -171,7 +171,7 @@ for pr in ${pr_list}; do
echo ""
} >> "${output_ci_single}"
first_line=$(head -n 1 "${output_ci_single}")
- if [[ "$first_line" != '```' ]]; then
+ if [[ "${first_line}" != '```' ]]; then
sed -i "1 i\`\`\`" "${output_ci_single}"
fi
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
--
GitLab
From b5a6ca8b17b264a5a34f80488a9c63990b9d665e Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 12:05:17 -0500
Subject: [PATCH 061/189] more bash norms quotes
---
ci/scripts/utils/ci_utils.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 01122c1b3..ffee1b55b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -142,9 +142,9 @@ function publish_logs() {
if [[ -f "${full_path}" ]]; then
full_paths+="${full_path} "
else
- echo "File "${full_path}" does not exist"
+ echo "File ${full_path} does not exist"
fi
- done < "$file"
+ done < "${file}"
if [[ -n "${full_paths}" ]]; then
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo "${PR_header}" > /dev/null
--
GitLab
From 90359034139cf2c5d8829d5e0a529c59f78e8e1b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 19:24:43 +0000
Subject: [PATCH 062/189] updated file type to input argument
---
ci/scripts/utils/rocoto_statcount.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index 91f8ca4da..fd34e88dd 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -4,7 +4,7 @@ import sys
import os
from wxflow import Executable, which, Logger, CommandNotFoundError
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from argparse import ArgumentParser, FileType
logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
@@ -28,8 +28,8 @@ def input_args():
parser = ArgumentParser(description=description)
- parser.add_argument('-w', help='workflow_document', nargs=1, metavar='XML Workflow File', type=str, required=True)
- parser.add_argument('-d', help='database_file', nargs=1, metavar='Database File', type=str, required=True)
+ parser.add_argument('-w', help='workflow_document', nargs=1, metavar='XML Workflow File', type=FileType('r'), required=True)
+ parser.add_argument('-d', help='database_file', nargs=1, metavar='Database File', type=FileType('r'), required=True)
parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
args = parser.parse_args()
--
GitLab
From 617ec380481ba77d9175a6c6e15724a3a97f54db Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 19:31:26 +0000
Subject: [PATCH 063/189] updated file type name
---
ci/scripts/utils/rocoto_statcount.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index fd34e88dd..ba70f0283 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -50,8 +50,8 @@ def rocoto_statcount():
logger.exception("rocotostat not found in PATH")
raise CommandNotFoundError("rocotostat not found in PATH")
- xml_file_path = os.path.abspath(args.w)
- db_file_path = os.path.abspath(args.d)
+ xml_file_path = os.path.abspath(args.w.name)
+ db_file_path = os.path.abspath(args.d.name)
rocotostat_all = which("rocotostat")
rocotostat.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-s'])
--
GitLab
From 916892a034d94d1e9b2b555ff4b57cd7633c6ee8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 19:42:23 +0000
Subject: [PATCH 064/189] fixed name of arg to files in rocoto stat
---
ci/scripts/utils/rocoto_statcount.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index ba70f0283..dbdfab7b6 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -28,8 +28,8 @@ def input_args():
parser = ArgumentParser(description=description)
- parser.add_argument('-w', help='workflow_document', nargs=1, metavar='XML Workflow File', type=FileType('r'), required=True)
- parser.add_argument('-d', help='database_file', nargs=1, metavar='Database File', type=FileType('r'), required=True)
+ parser.add_argument('-w', help='workflow_document', type=FileType('r'), required=True)
+ parser.add_argument('-d', help='database_file', metavar='Database File', type=FileType('r'), required=True)
parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
args = parser.parse_args()
@@ -54,8 +54,8 @@ def rocoto_statcount():
db_file_path = os.path.abspath(args.d.name)
rocotostat_all = which("rocotostat")
- rocotostat.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-s'])
- rocotostat_all.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-a'])
+ rocotostat.add_default_arg(['-w',os.path.abspath(args.w.name), '-d', db_file_path, '-s'])
+ rocotostat_all.add_default_arg(['-w',os.path.abspath(args.d.name), '-d', db_file_path, '-a'])
rocotostat_output = rocotostat(output=str)
rocotostat_output = rocotostat_output.splitlines()[1:]
--
GitLab
From 7c86ea9c8aaec6b31b89876a5142caff197b7671 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 14:51:34 -0500
Subject: [PATCH 065/189] fixed args and extra space on verbose output
---
ci/scripts/utils/rocoto_statcount.py | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index dbdfab7b6..2ded40881 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -50,12 +50,9 @@ def rocoto_statcount():
logger.exception("rocotostat not found in PATH")
raise CommandNotFoundError("rocotostat not found in PATH")
- xml_file_path = os.path.abspath(args.w.name)
- db_file_path = os.path.abspath(args.d.name)
-
rocotostat_all = which("rocotostat")
- rocotostat.add_default_arg(['-w',os.path.abspath(args.w.name), '-d', db_file_path, '-s'])
- rocotostat_all.add_default_arg(['-w',os.path.abspath(args.d.name), '-d', db_file_path, '-a'])
+ rocotostat.add_default_arg(['-w',os.path.abspath(args.w.name), '-d',os.path.abspath(args.d.name), '-s'])
+ rocotostat_all.add_default_arg(['-w',os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-a'])
rocotostat_output = rocotostat(output=str)
rocotostat_output = rocotostat_output.splitlines()[1:]
@@ -102,7 +99,7 @@ if __name__ == '__main__':
if args.verbose:
for status in rocoto_status:
- print(f'Number of {status} : {rocoto_status[status]}\n')
+ print(f'Number of {status} : {rocoto_status[status]}')
print(rocoto_state)
sys.exit(error_return)
--
GitLab
From d3e657dd9ca9524461eea1b232011737d78e07ac Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 20:06:30 +0000
Subject: [PATCH 066/189] removed verspose and quotes to args for check rocoto
state
---
ci/scripts/check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index ec880e71a..75abc9331 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -125,7 +125,7 @@ for pr in ${pr_list}; do
fi
set +e
- rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db} --verbose")
+ rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w ${xml} -d ${db})
rocoto_error=$?
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
--
GitLab
From bd703881f3e8854bda5b25f15214f6ce6ca85b59 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 15:21:20 -0500
Subject: [PATCH 067/189] fixed shellnorms on quotes for rocoto stat
---
ci/scripts/check_ci.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 75abc9331..b4b58d35e 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -14,7 +14,8 @@ echo "Begin ${scriptname} at $(date -u)" || true
export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
GH=${HOME}/bin/gh
-REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
+#REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
+REPO_URL="git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git"
#########################################################################
# Set up runtime environment varibles for accounts on supproted machines
@@ -125,7 +126,7 @@ for pr in ${pr_list}; do
fi
set +e
- rocoto_state=$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w ${xml} -d ${db})
+ rocoto_state="$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}")"
rocoto_error=$?
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
--
GitLab
From f822c8ee05c0c7724f676be46990649f617bb012 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 20:26:13 +0000
Subject: [PATCH 068/189] pycodestyle spaces
---
ci/scripts/utils/rocoto_statcount.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index 2ded40881..c9284f62a 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -51,8 +51,8 @@ def rocoto_statcount():
raise CommandNotFoundError("rocotostat not found in PATH")
rocotostat_all = which("rocotostat")
- rocotostat.add_default_arg(['-w',os.path.abspath(args.w.name), '-d',os.path.abspath(args.d.name), '-s'])
- rocotostat_all.add_default_arg(['-w',os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-a'])
+ rocotostat.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-s'])
+ rocotostat_all.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-a'])
rocotostat_output = rocotostat(output=str)
rocotostat_output = rocotostat_output.splitlines()[1:]
--
GitLab
From 004449951194c33a45799e34e2c60bd756a925af Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 15:29:45 -0500
Subject: [PATCH 069/189] pycodestyle white space
---
workflow/rocoto_viewer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/workflow/rocoto_viewer.py b/workflow/rocoto_viewer.py
index 78936c720..459381f60 100755
--- a/workflow/rocoto_viewer.py
+++ b/workflow/rocoto_viewer.py
@@ -1360,7 +1360,7 @@ def main(screen):
screen.refresh()
curses.mousemask(1)
curses.noecho()
- for i in range(0, curses.COLORS-1):
+ for i in range(0, curses.COLORS - 1):
curses.init_pair(i + 1, i, curses.COLOR_BLACK)
if i == 4:
curses.init_pair(i + 1, i, curses.COLOR_WHITE)
--
GitLab
From eb470bbd5cda93c79a04bee8dc0b0d6035b3422e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 20:32:37 +0000
Subject: [PATCH 070/189] added -k argument to build_all.sh in clone-build for
CI to capture error.logs
---
ci/scripts/clone-build_ci.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
index 989afabb8..7b3c1176d 100755
--- a/ci/scripts/clone-build_ci.sh
+++ b/ci/scripts/clone-build_ci.sh
@@ -74,7 +74,7 @@ set +e
source "${HOMEgfs}/ush/module-setup.sh"
export BUILD_JOBS=8
rm -rf log.build
-./build_all.sh -guw >> log.build 2>&1
+./build_all.sh -kguw >> log.build 2>&1
build_status=$?
DATE=$(date +'%D %r')
@@ -83,6 +83,7 @@ if [[ ${build_status} != 0 ]]; then
echo "Build: *** FAILED ***"
echo "Build: Failed at ${DATE}"
cat "${PWD}/log.build"
+ cat "${PWD}/logs/error.logs"
} >> "${outfile}"
exit "${build_status}"
else
--
GitLab
From badd76317825237ba30b8d7eeb5434ea892a0409 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 15:41:30 -0500
Subject: [PATCH 071/189] SC2312 is false negative for kill commands
---
ci/scripts/driver.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index a50ba3620..41d7bd98b 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -101,8 +101,8 @@ for pr in ${pr_list}; do
echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
fi
else
- # shellcheck disable=SC2312
ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
+ # shellcheck enable=SC2312
fi
{
echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
--
GitLab
From 448f9c7d5c29b9e2311a4f9dae9a229c257cc968 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 15:54:18 -0500
Subject: [PATCH 072/189] fixed needed quotes for shellchecks
---
ci/scripts/driver.sh | 4 ++--
ci/scripts/utils/ci_utils.sh | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 41d7bd98b..a7be30764 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -94,16 +94,16 @@ for pr in ${pr_list}; do
} >> "${output_ci_single}"
if [[ "${driver_PID}" -ne 0 ]]; then
echo "Driver PID: ${driver_PID} no longer running this build having it killed"
+ # shellcheck disable=SC2312
if [[ "${driver_HOST}" == "${host_name}" ]]; then
- # shellcheck disable=SC2312
pstree_out=$(pstree -A -p "${driver_PID}")
if [[ -n "${pstree_out}" ]]; then
echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
fi
else
ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
- # shellcheck enable=SC2312
fi
+ # shellcheck enable=SC2312
{
echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
echo "Driver PID: has restarted as $$ on ${host_name}"
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index ffee1b55b..0a1d21f47 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -147,8 +147,8 @@ function publish_logs() {
done < "${file}"
if [[ -n "${full_paths}" ]]; then
- ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo "${PR_header}" > /dev/null
- URL=$(${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --gist "${PR_header}")
+ "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file "${full_paths}" --repo "${PR_header}"" > /dev/null
+ URL="$(${HOMEgfs}/ci/scripts/utils/publish_logs.py --file "${full_paths}" --gist "${PR_header}")"
fi
echo "${URL}"
}
--
GitLab
From d08a43d156e90e3a44d9da4b26c7a5a287c147d0 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 16:01:22 -0500
Subject: [PATCH 073/189] SC override issues
---
ci/scripts/driver.sh | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index a7be30764..b6978466c 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -94,16 +94,14 @@ for pr in ${pr_list}; do
} >> "${output_ci_single}"
if [[ "${driver_PID}" -ne 0 ]]; then
echo "Driver PID: ${driver_PID} no longer running this build having it killed"
- # shellcheck disable=SC2312
if [[ "${driver_HOST}" == "${host_name}" ]]; then
- pstree_out=$(pstree -A -p "${driver_PID}")
+ pstree_out="$(pstree -A -p "${driver_PID}")"
if [[ -n "${pstree_out}" ]]; then
echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
fi
else
ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
fi
- # shellcheck enable=SC2312
{
echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
echo "Driver PID: has restarted as $$ on ${host_name}"
--
GitLab
From 808b3598e8b3341a67f50c2be4b289d3f103c2ae Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 16:12:01 -0500
Subject: [PATCH 074/189] SC2027 and 2086 disable
---
ci/scripts/utils/ci_utils.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 0a1d21f47..abc125545 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -147,7 +147,8 @@ function publish_logs() {
done < "${file}"
if [[ -n "${full_paths}" ]]; then
- "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file "${full_paths}" --repo "${PR_header}"" > /dev/null
+ # shellcheck disable=SC2027,SC2086
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null
URL="$(${HOMEgfs}/ci/scripts/utils/publish_logs.py --file "${full_paths}" --gist "${PR_header}")"
fi
echo "${URL}"
--
GitLab
From 5c35e7f78e8e7ff21bb4ef55bc0410462e7b3fe0 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 16:18:25 -0500
Subject: [PATCH 075/189] still more and more quotes
---
ci/scripts/utils/ci_utils.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index abc125545..72b496207 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -149,7 +149,7 @@ function publish_logs() {
if [[ -n "${full_paths}" ]]; then
# shellcheck disable=SC2027,SC2086
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null
- URL="$(${HOMEgfs}/ci/scripts/utils/publish_logs.py --file "${full_paths}" --gist "${PR_header}")"
+ URL="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")"
fi
echo "${URL}"
}
--
GitLab
From e1eb68348d75270976481e63067c7dbda012a62a Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 3 Apr 2024 16:23:38 -0500
Subject: [PATCH 076/189] disable SC312 because can not decoplse of quote kill
command
---
ci/scripts/driver.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index b6978466c..a94c72fac 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -97,7 +97,8 @@ for pr in ${pr_list}; do
if [[ "${driver_HOST}" == "${host_name}" ]]; then
pstree_out="$(pstree -A -p "${driver_PID}")"
if [[ -n "${pstree_out}" ]]; then
- echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
+ #shellcheck disable=SC2312
+ echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
fi
else
ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
--
GitLab
From ab9cc4101497300f4cdc6cda17db876f2ebc4964 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 14:02:49 +0000
Subject: [PATCH 077/189] added a short verbose called v
---
ci/scripts/utils/rocoto_statcount.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocoto_statcount.py
index c9284f62a..b73ddc5f1 100755
--- a/ci/scripts/utils/rocoto_statcount.py
+++ b/ci/scripts/utils/rocoto_statcount.py
@@ -31,6 +31,7 @@ def input_args():
parser.add_argument('-w', help='workflow_document', type=FileType('r'), required=True)
parser.add_argument('-d', help='database_file', metavar='Database File', type=FileType('r'), required=True)
parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
+ parser.add_argument('-v', action='store_true', help='List the states and the number of jobs that are in each', required=False)
args = parser.parse_args()
@@ -97,9 +98,12 @@ if __name__ == '__main__':
else:
rocoto_state = 'RUNNING'
- if args.verbose:
+ if args.verbose or args.v:
for status in rocoto_status:
- print(f'Number of {status} : {rocoto_status[status]}')
+ if args.v:
+ print(f'{status}:{rocoto_status[status]}')
+ else:
+ print(f'Number of {status} : {rocoto_status[status]}')
print(rocoto_state)
sys.exit(error_return)
--
GitLab
From a72aa6b1d004d1cfb16c66716618033206316754 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 16:42:51 +0000
Subject: [PATCH 078/189] working on check test
---
ci/scripts/run-check_ci.sh | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 8e1e92705..95615d42c 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -4,11 +4,13 @@ set -eu
#####################################################################################
# Script description: script to check the status of an experiment as reported
-# by Rocoto
+# by Rocoto
#####################################################################################
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
+ROOT_DIR=${3:-${ROOT_DIR:-?}} # Location of the root of the global-workflow repository
+ # that runs CI functions that must be outside of PRs
# TEST_DIR contains 2 directories;
# 1. HOMEgfs: clone of the global-workflow
@@ -66,13 +68,18 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
- rocotostat_output=$(rocotostat -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true
- num_cycles=$(echo "${rocotostat_output}" | wc -l) || true
- num_done=$(echo "${rocotostat_output}" | grep -c Done) || true
- num_succeeded=$(rocotostat -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
- num_failed=$(rocotostat -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true
- echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}"
+ $
+
+ rocotostat_output="$(${ROOT_DIR}/ci/scripts/utils/rocotostat.py -w "${xml}" -d "${db}" -v)" || true
+ num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
+ num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done) | cut -d: -f2) || true
+ num_succeeded=$(echo "${rocotostat_output}" | grep SUCCEEDED) | cut -d: -f2) || true
+ num_failed=$(echo "${rocotostat_output}" | grep FAIL) | cut -d: -f2) || true
+
+ echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_cycles_done}"
+
+ exit 0
if [[ ${num_failed} -ne 0 ]]; then
{
@@ -84,7 +91,7 @@ while true; do
echo "Error logs:"
echo "${error_logs}"
} | tee -a "${run_check_logfile}"
- # rm -f "${RUNTESTS}/error.logs"
+ rm -f "${RUNTESTS}/error.logs"
for log in ${error_logs}; do
echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/error.logs"
done
--
GitLab
From a83cc5e74246f8d7c2a98f1f9ec4469c2c0906de Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 16:48:13 +0000
Subject: [PATCH 079/189] working on check test
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 95615d42c..35e810e4b 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -25,7 +25,7 @@ ROOT_DIR=${3:-${ROOT_DIR:-?}} # Location of the root of the global-workflow rep
# └── ${pslot}
# Two system build directories created at build time gfs, and gdas
# TODO: Make this configurable (for now all scripts run from gfs for CI at runtime)
-HOMEgfs="${TEST_DIR}/gfs"
+HOMEgfs="${TEST_DIR}/${SYSTEM_BUILD_DIR:-global-workflow}"
RUNTESTS="${TEST_DIR}/RUNTESTS"
run_check_logfile="${RUNTESTS}/ci-run_check.log"
--
GitLab
From ba2e4bc64796ac71fcdd96c02ae87ba764dd009a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 16:53:04 +0000
Subject: [PATCH 080/189] working on check test
---
ci/scripts/run-check_ci.sh | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 35e810e4b..7901797d4 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,12 +73,14 @@ while true; do
rocotostat_output="$(${ROOT_DIR}/ci/scripts/utils/rocotostat.py -w "${xml}" -d "${db}" -v)" || true
num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
- num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done) | cut -d: -f2) || true
- num_succeeded=$(echo "${rocotostat_output}" | grep SUCCEEDED) | cut -d: -f2) || true
- num_failed=$(echo "${rocotostat_output}" | grep FAIL) | cut -d: -f2) || true
+ num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done | cut -d: -f2) || true
+ num_succeeded=$(echo "${rocotostat_output}" | grep SUCCEEDED | cut -d: -f2) || true
+ num_failed=$(echo "${rocotostat_output}" | grep FAIL | cut -d: -f2) || true
echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_cycles_done}"
+ echo $num_succeeded $num_failed
+
exit 0
if [[ ${num_failed} -ne 0 ]]; then
--
GitLab
From 8a95e8f3c284329a4c017fb0b7bfd8b67cfd81ee Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 13:49:07 -0500
Subject: [PATCH 081/189] finished debugging run check with stall feature
---
ci/scripts/run-check_ci.sh | 53 +++++++++----------
.../{rocoto_statcount.py => rocotostat.py} | 0
2 files changed, 26 insertions(+), 27 deletions(-)
rename ci/scripts/utils/{rocoto_statcount.py => rocotostat.py} (100%)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 7901797d4..499e1ba2a 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -49,8 +49,8 @@ fi
# Launch experiment
echo "Launch experiment with Rocoto."
-rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
-sleep 30
+#rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
+#sleep 30
if [[ ! -f "${db}" ]]; then
echo "FATAL ERROR: Rocoto database file ${db} not found, experiment ${pslot} failed, ABORT!"
exit 2
@@ -58,52 +58,51 @@ fi
# Experiment launched
rc=99
+set +e
while true; do
echo "Run rocotorun."
- rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
+ #rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
# Wait before running rocotostat
- sleep 30
+ #sleep 30
# Get job statistics
echo "Gather Rocoto statistics"
+ rocotostat_output="$(${ROOT_DIR}/ci/scripts/utils/rocotostat.py -w "${xml}" -d "${db}" -v)"
+ error_stat=$?
- $
-
- rocotostat_output="$(${ROOT_DIR}/ci/scripts/utils/rocotostat.py -w "${xml}" -d "${db}" -v)" || true
num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done | cut -d: -f2) || true
num_succeeded=$(echo "${rocotostat_output}" | grep SUCCEEDED | cut -d: -f2) || true
num_failed=$(echo "${rocotostat_output}" | grep FAIL | cut -d: -f2) || true
+ rocoto_stat=$(echo "${rocotostat_output}" | tail -1) || true
echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_cycles_done}"
- echo $num_succeeded $num_failed
-
- exit 0
-
- if [[ ${num_failed} -ne 0 ]]; then
+ if [[ ${error_stat} -ne 0 ]]; then
{
- echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
- echo "Experiment ${pslot} Terminated: *FAILED*"
+ echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed or dead at $(date)" || true
+ echo "Experiment ${pslot} Terminated: *${rocoto_stat}*"
} | tee -a "${run_check_logfile}"
- error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- {
- echo "Error logs:"
- echo "${error_logs}"
- } | tee -a "${run_check_logfile}"
- rm -f "${RUNTESTS}/error.logs"
- for log in ${error_logs}; do
- echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/error.logs"
- done
- rc=1
- break
+ if [[ "${num_failed}" -ne 0 ]]; then
+ error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
+ {
+ echo "Error logs:"
+ echo "${error_logs}"
+ } | tee -a "${run_check_logfile}"
+ rm -f "${RUNTESTS}/error.logs"
+ for log in ${error_logs}; do
+ echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/error.logs"
+ done
+ fi
+ rc=1
+ break
fi
- if [[ "${num_done}" -eq "${num_cycles}" ]]; then
+ if [[ "${rocoto_stat}" == "DONE" ]]; then
{
- echo "Experiment ${pslot} Completed at $(date)" || true
+ echo "Experiment ${pslot} Completed ${num_cycles_done} Cycles at $(date)" || true
echo "with ${num_succeeded} successfully completed jobs" || true
echo "Experiment ${pslot} Completed: *SUCCESS*"
} | tee -a "${run_check_logfile}"
diff --git a/ci/scripts/utils/rocoto_statcount.py b/ci/scripts/utils/rocotostat.py
similarity index 100%
rename from ci/scripts/utils/rocoto_statcount.py
rename to ci/scripts/utils/rocotostat.py
--
GitLab
From 91f6b704fdcda82bd013f94d3bdfbbee3945b729 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 19:01:00 +0000
Subject: [PATCH 082/189] renamed rocoto_statcount to shorter rocotostat.py
---
ci/scripts/check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index b4b58d35e..112b3d21f 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -126,7 +126,7 @@ for pr in ${pr_list}; do
fi
set +e
- rocoto_state="$("${HOMEgfs}/ci/scripts/utils/rocoto_statcount.py" -w "${xml}" -d "${db}")"
+ rocoto_state="$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}")"
rocoto_error=$?
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
--
GitLab
From 1ae92c8227d98d278f349c37113f93148cc43809 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 15:01:07 -0500
Subject: [PATCH 083/189] finilized updated run-check for Jenknins with STALL
feature
---
ci/scripts/run-check_ci.sh | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 499e1ba2a..956eebf0b 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -49,8 +49,8 @@ fi
# Launch experiment
echo "Launch experiment with Rocoto."
-#rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
-#sleep 30
+rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
+sleep 30
if [[ ! -f "${db}" ]]; then
echo "FATAL ERROR: Rocoto database file ${db} not found, experiment ${pslot} failed, ABORT!"
exit 2
@@ -62,10 +62,10 @@ set +e
while true; do
echo "Run rocotorun."
- #rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
+ rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
# Wait before running rocotostat
- #sleep 30
+ sleep 30
# Get job statistics
echo "Gather Rocoto statistics"
@@ -78,7 +78,7 @@ while true; do
num_failed=$(echo "${rocotostat_output}" | grep FAIL | cut -d: -f2) || true
rocoto_stat=$(echo "${rocotostat_output}" | tail -1) || true
- echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_cycles_done}"
+ echo "(${pslot}) Total Cycles: ${num_cycles} number done: ${num_cycles_done} ${rocoto_stat} on ${MACHINE_ID^}"
if [[ ${error_stat} -ne 0 ]]; then
{
--
GitLab
From 129b38ec2852e0a245a60f739d139678b7d39476 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 20:07:58 +0000
Subject: [PATCH 084/189] shellnorms quotes on rocotostat script
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 956eebf0b..a6a208c05 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -69,7 +69,7 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
- rocotostat_output="$(${ROOT_DIR}/ci/scripts/utils/rocotostat.py -w "${xml}" -d "${db}" -v)"
+ rocotostat_output="$("${ROOT_DIR}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)"
error_stat=$?
num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
--
GitLab
From 71491bddd5a11d3e15f996f8d29ba5e0b89dcf0b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 20:46:22 +0000
Subject: [PATCH 085/189] added a gh case for changing a label and a PR label
in Jekins as not to use pullRequest
---
ci/Jenkinsfile | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index c799d3c48..7aca39bd3 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -46,10 +46,12 @@ pipeline {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
- pullRequest.addLabel("CI-${Machine}-Building")
- if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
- pullRequest.removeLabel("CI-${Machine}-Ready")
- }
+ // pullRequest.addLabel("CI-${Machine}-Building")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${GIT_URL} --add-label "CI-${Machine}-Building" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${GIT_URL} --remove-label "CI-${Machine}-Ready" """)
+ //if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
+ // pullRequest.removeLabel("CI-${Machine}-Ready")
+ //}
}
echo "Building and running on ${Machine} in directory ${HOME}"
}
@@ -204,7 +206,9 @@ pipeline {
repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim()
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
try {
- pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
+ // pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${GIT_URL} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
+
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
--
GitLab
From 0e5ac222376137946e3f30f0ec7774357dfa1cec Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 4 Apr 2024 21:49:50 +0000
Subject: [PATCH 086/189] replaced all the pullRequests with GH commands in the
Jenkkinsfile
---
ci/Jenkinsfile | 41 +++++++++++++++++++++++++----------------
1 file changed, 25 insertions(+), 16 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 7aca39bd3..18e4d7bb9 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -4,6 +4,7 @@ def HOME = 'none'
def caseList = ''
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
+def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
pipeline {
agent { label 'built-in' }
@@ -47,8 +48,8 @@ pipeline {
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
// pullRequest.addLabel("CI-${Machine}-Building")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${GIT_URL} --add-label "CI-${Machine}-Building" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${GIT_URL} --remove-label "CI-${Machine}-Ready" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Ready" """)
//if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
// pullRequest.removeLabel("CI-${Machine}-Ready")
//}
@@ -126,10 +127,12 @@ pipeline {
}
if (env.CHANGE_ID && system == 'gfs') {
try {
- if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
- pullRequest.removeLabel("CI-${Machine}-Building")
- }
- pullRequest.addLabel("CI-${Machine}-Running")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ //if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
+ // pullRequest.removeLabel("CI-${Machine}-Building")
+ //}
+ //pullRequest.addLabel("CI-${Machine}-Running")
} catch (Exception e) {
echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
}
@@ -207,7 +210,7 @@ pipeline {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
try {
// pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${GIT_URL} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -231,11 +234,13 @@ pipeline {
script {
if(env.CHANGE_ID) {
try {
- for (label in pullRequest.labels) {
- if (label.contains("${Machine}")) {
- pullRequest.removeLabel(label)
- }
- }
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ //for (label in pullRequest.labels) {
+ // if (label.contains("${Machine}")) {
+ // pullRequest.removeLabel(label)
+ // }
+ //}
} catch (Exception e) {
echo "Failed to remove labels: ${e.getMessage()}"
}
@@ -246,9 +251,11 @@ pipeline {
script {
if(env.CHANGE_ID) {
try {
- pullRequest.addLabel("CI-${Machine}-Passed")
+ //pullRequest.addLabel("CI-${Machine}-Passed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+ //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
} catch (Exception e) {
echo "Failed to add success label or comment: ${e.getMessage()}"
}
@@ -259,9 +266,11 @@ pipeline {
script {
if(env.CHANGE_ID) {
try {
- pullRequest.addLabel("CI-${Machine}-Failed")
+ //pullRequest.addLabel("CI-${Machine}-Failed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+ //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
} catch (Exception e) {
echo "Failed to add failure label or comment: ${e.getMessage()}"
}
--
GitLab
From 8eff7e93b78b196cc5d49e92e65ac0d3be7fcc3c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 00:59:32 +0000
Subject: [PATCH 087/189] needed to set new env var SYSTEM_BUILD_DIR
---
ci/Jenkinsfile | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 18e4d7bb9..f73375c10 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -47,12 +47,8 @@ pipeline {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
- // pullRequest.addLabel("CI-${Machine}-Building")
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" """)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Ready" """)
- //if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
- // pullRequest.removeLabel("CI-${Machine}-Ready")
- //}
}
echo "Building and running on ${Machine} in directory ${HOME}"
}
@@ -129,10 +125,6 @@ pipeline {
try {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" """)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- //if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
- // pullRequest.removeLabel("CI-${Machine}-Building")
- //}
- //pullRequest.addLabel("CI-${Machine}-Running")
} catch (Exception e) {
echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
}
@@ -183,6 +175,7 @@ pipeline {
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
+ env.SYSTEM_BUILD_DIR=system // used to set the system build directory in the run scripts
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
@@ -209,7 +202,6 @@ pipeline {
repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim()
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
try {
- // pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
} catch (Exception error_comment) {
@@ -230,6 +222,7 @@ pipeline {
}
post {
+ agent { label "${machine}-emc" }
always {
script {
if(env.CHANGE_ID) {
--
GitLab
From e6eca8b117860f501357aeb64a1cacf009a94190 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 01:34:30 +0000
Subject: [PATCH 088/189] added gist pup to check_ci for bash for experment
error logs
---
ci/scripts/check_ci.sh | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 112b3d21f..a836f6711 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -139,13 +139,17 @@ for pr in ${pr_list}; do
# TODO used rocotocheck to find the missing dependency
else
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
+ gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${error_logs}" --gist "PR_${pr}")"
{
- echo "Experiment ${pslot} *** ${rocoto_state} *** on ${MACHINE_ID^}"
+ echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}"
echo "Experiment ${pslot} with ${rocoto_state} tasks failed at $(date +'%D %r')" || true
echo "Error logs:"
+ echo "```"
echo "${error_logs}"
+ echo "```"
+ echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})"
} >> "${output_ci}"
- sed -i "1 i\`\`\`" "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
fi
"${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
--
GitLab
From b917feec489c17c4a7dc4c9ba736df18f4f48483 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 02:47:20 +0000
Subject: [PATCH 089/189] shellnorms quote crap buch are wrong like tics and
such
---
ci/scripts/check_ci.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index a836f6711..af22e78e8 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -139,14 +139,14 @@ for pr in ${pr_list}; do
# TODO used rocotocheck to find the missing dependency
else
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
+ "${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${error_logs}" --repo "PR_${pr}" > /dev/null
gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${error_logs}" --gist "PR_${pr}")"
{
echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}"
echo "Experiment ${pslot} with ${rocoto_state} tasks failed at $(date +'%D %r')" || true
echo "Error logs:"
echo "```"
- echo "${error_logs}"
+ echo ${error_logs}
echo "```"
echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})"
} >> "${output_ci}"
--
GitLab
From d00a5ef3f4714e5689e8166ef92c33c260ba6788 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 02:49:56 +0000
Subject: [PATCH 090/189] trying to escape tics for log file markup to GitHub
messages
---
ci/scripts/check_ci.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index af22e78e8..711ee3857 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -145,9 +145,9 @@ for pr in ${pr_list}; do
echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}"
echo "Experiment ${pslot} with ${rocoto_state} tasks failed at $(date +'%D %r')" || true
echo "Error logs:"
- echo "```"
+ echo "\`\`\`"
echo ${error_logs}
- echo "```"
+ echo "\`\`\`"
echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})"
} >> "${output_ci}"
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
--
GitLab
From 768707521200e782f02bb97a9ce9b8f81731b0af Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 12:58:53 +0000
Subject: [PATCH 091/189] moved post into a stage of the main pipeline so it
can use GH on host machine too
---
ci/Jenkinsfile | 98 ++++++++++++++++++++++++++++----------------------
1 file changed, 55 insertions(+), 43 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index f73375c10..a6abbca9e 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -2,6 +2,7 @@ def Machine = 'none'
def machine = 'none'
def HOME = 'none'
def caseList = ''
+def FINAL_STATE = 'PASSED'
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
@@ -115,10 +116,11 @@ pipeline {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
error("Failed to build system on ${Machine}")
+ FINAL_STATE='FAILED'
}
}
sh(script: './link_workflow.sh')
- // sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
}
}
if (env.CHANGE_ID && system == 'gfs') {
@@ -211,6 +213,7 @@ pipeline {
echo "No error logs found for failed cases in $HOME/RUNTESTS/error.logs"
}
error("Failed to run experiments ${Case} on ${Machine}")
+ FINAL_STATE='FAILED'
}
}
}
@@ -219,56 +222,65 @@ pipeline {
}
}
}
- }
- post {
- agent { label "${machine}-emc" }
- always {
- script {
- if(env.CHANGE_ID) {
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- //for (label in pullRequest.labels) {
- // if (label.contains("${Machine}")) {
- // pullRequest.removeLabel(label)
- // }
- //}
- } catch (Exception e) {
- echo "Failed to remove labels: ${e.getMessage()}"
+ stage('Finalize') {
+ agent { label "${machine}-emc" }
+ stages {
+ stage ('Always') {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ //for (label in pullRequest.labels) {
+ // if (label.contains("${Machine}")) {
+ // pullRequest.removeLabel(label)
+ // }
+ //}
+ } catch (Exception e) {
+ echo "Failed to remove labels: ${e.getMessage()}"
+ }
+ }
}
}
- }
- }
- success {
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Passed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add success label or comment: ${e.getMessage()}"
+ stage ('Success') {
+ when {
+ expression { return FINAL_STATE == 'PASSED' }
+ }
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ //pullRequest.addLabel("CI-${Machine}-Passed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add success label or comment: ${e.getMessage()}"
+ }
+ }
}
}
- }
- }
- failure {
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Failed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add failure label or comment: ${e.getMessage()}"
+ stage('Failure') {
+ when {
+ expression { return FINAL_STATE == 'FAILED' }
+ }
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ //pullRequest.addLabel("CI-${Machine}-Failed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add failure label or comment: ${e.getMessage()}"
+ }
+ }
}
}
}
}
}
+
}
--
GitLab
From 50f817e664f809b8be1e23566e8d790f88f7ebd8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 13:19:28 +0000
Subject: [PATCH 092/189] Stages in Finilized needed steps
---
ci/Jenkinsfile | 70 +++++++++++++++++++++++++++-----------------------
1 file changed, 38 insertions(+), 32 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index a6abbca9e..af8ae01e2 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -227,18 +227,20 @@ pipeline {
agent { label "${machine}-emc" }
stages {
stage ('Always') {
- script {
- if(env.CHANGE_ID) {
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- //for (label in pullRequest.labels) {
- // if (label.contains("${Machine}")) {
- // pullRequest.removeLabel(label)
- // }
- //}
- } catch (Exception e) {
- echo "Failed to remove labels: ${e.getMessage()}"
+ steps {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ //for (label in pullRequest.labels) {
+ // if (label.contains("${Machine}")) {
+ // pullRequest.removeLabel(label)
+ // }
+ //}
+ } catch (Exception e) {
+ echo "Failed to remove labels: ${e.getMessage()}"
+ }
}
}
}
@@ -247,16 +249,18 @@ pipeline {
when {
expression { return FINAL_STATE == 'PASSED' }
}
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Passed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add success label or comment: ${e.getMessage()}"
+ steps {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ //pullRequest.addLabel("CI-${Machine}-Passed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add success label or comment: ${e.getMessage()}"
+ }
}
}
}
@@ -265,16 +269,18 @@ pipeline {
when {
expression { return FINAL_STATE == 'FAILED' }
}
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Failed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add failure label or comment: ${e.getMessage()}"
+ steps {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ //pullRequest.addLabel("CI-${Machine}-Failed")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add failure label or comment: ${e.getMessage()}"
+ }
}
}
}
--
GitLab
From b140924195c109c9ea1337a98d0212296596cb4b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 14:51:09 +0000
Subject: [PATCH 093/189] need to not use ROOT_DIR when ci scripts run from PR
as with Jenkins
---
ci/scripts/run-check_ci.sh | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index a6a208c05..c593e73f6 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -10,7 +10,7 @@ set -eu
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
ROOT_DIR=${3:-${ROOT_DIR:-?}} # Location of the root of the global-workflow repository
- # that runs CI functions that must be outside of PRs
+ # that runs CI functions that may be outside of PRs
# TEST_DIR contains 2 directories;
# 1. HOMEgfs: clone of the global-workflow
@@ -29,6 +29,10 @@ HOMEgfs="${TEST_DIR}/${SYSTEM_BUILD_DIR:-global-workflow}"
RUNTESTS="${TEST_DIR}/RUNTESTS"
run_check_logfile="${RUNTESTS}/ci-run_check.log"
+if [[ "${ROOT_DIR}" == "?" ]]; then
+ ROOT_DIR="${HOMEgfs}"
+fi
+
# Source modules and setup logging
echo "Source modules."
source "${HOMEgfs}/workflow/gw_setup.sh"
--
GitLab
From 03967bb2e3ce8b351ac24277eaf6f6eccb208b1f Mon Sep 17 00:00:00 2001
From: "Terry.McGuinness" <Terry.McGuinness@noaa.gov>
Date: Fri, 5 Apr 2024 16:37:28 +0000
Subject: [PATCH 094/189] fixed up printing and publishing error logs in check
script
---
ci/scripts/check_ci.sh | 32 +++++++++++---------------------
1 file changed, 11 insertions(+), 21 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 711ee3857..54c17d63b 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -130,34 +130,24 @@ for pr in ${pr_list}; do
rocoto_error=$?
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
- "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
- # Check if the experiment failed due to a missing dependency and is stalled
- if [[ "${rocoto_state}" == "STALLED" ]]; then
- date=$(date +'%D %r')
- echo "Experiment ${pslot} with **${rocoto_state}** at ${date} on ${MACHINE_ID^}" >> "${output_ci_single}"
- "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
- # TODO used rocotocheck to find the missing dependency
- else
+ "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- "${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${error_logs}" --repo "PR_${pr}" > /dev/null
- gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${error_logs}" --gist "PR_${pr}")"
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
+ gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")"
{
- echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^}"
- echo "Experiment ${pslot} with ${rocoto_state} tasks failed at $(date +'%D %r')" || true
+ echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')"
+ echo ""
echo "Error logs:"
echo "\`\`\`"
- echo ${error_logs}
+ echo "${error_logs}"
echo "\`\`\`"
echo "Follow link here to view the contents of the above file(s): [(link)](${gist_url})"
- } >> "${output_ci}"
- "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
- fi
- "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
- cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
- exit "${rocoto_error}"
+ } >> "${output_ci_single}"
+ "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
+ "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
+ exit "${rocoto_error}"
fi
- echo "${rocoto_state}"
- rocoto_state=$(echo -e "${rocoto_state}" | tail -1)
if [[ "${rocoto_state}" == "DONE" ]]; then
#Remove Experment cases that completed successfully
rm -Rf "${pslot_dir}"
--
GitLab
From f1df84d54d2fb75199570d02f1f884f851ea8271 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 16:59:05 +0000
Subject: [PATCH 095/189] moved fail fast
---
ci/Jenkinsfile | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index af8ae01e2..2808cfdef 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -12,7 +12,7 @@ pipeline {
options {
skipDefaultCheckout()
- parallelsAlwaysFailFast()
+ //parallelsAlwaysFailFast()
}
stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
@@ -70,6 +70,9 @@ pipeline {
}
stages {
stage('build system') {
+ options {
+ parallelsAlwaysFailFast()
+ }
steps {
script {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
--
GitLab
From c5b9fdebf8eaff669b3ee7cfd74d52ffb73f7f9e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 20:22:04 +0000
Subject: [PATCH 096/189] put post back in
---
ci/Jenkinsfile | 105 ++++++++++++++++++++-----------------------------
1 file changed, 42 insertions(+), 63 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 2808cfdef..a77231130 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -2,7 +2,6 @@ def Machine = 'none'
def machine = 'none'
def HOME = 'none'
def caseList = ''
-def FINAL_STATE = 'PASSED'
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
@@ -12,7 +11,7 @@ pipeline {
options {
skipDefaultCheckout()
- //parallelsAlwaysFailFast()
+ parallelsAlwaysFailFast()
}
stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
@@ -70,9 +69,6 @@ pipeline {
}
stages {
stage('build system') {
- options {
- parallelsAlwaysFailFast()
- }
steps {
script {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
@@ -119,11 +115,10 @@ pipeline {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
error("Failed to build system on ${Machine}")
- FINAL_STATE='FAILED'
}
}
sh(script: './link_workflow.sh')
- sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ // sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
}
}
if (env.CHANGE_ID && system == 'gfs') {
@@ -216,7 +211,6 @@ pipeline {
echo "No error logs found for failed cases in $HOME/RUNTESTS/error.logs"
}
error("Failed to run experiments ${Case} on ${Machine}")
- FINAL_STATE='FAILED'
}
}
}
@@ -225,71 +219,56 @@ pipeline {
}
}
}
+ }
- stage('Finalize') {
- agent { label "${machine}-emc" }
- stages {
- stage ('Always') {
- steps {
- script {
- if(env.CHANGE_ID) {
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- //for (label in pullRequest.labels) {
- // if (label.contains("${Machine}")) {
- // pullRequest.removeLabel(label)
- // }
- //}
- } catch (Exception e) {
- echo "Failed to remove labels: ${e.getMessage()}"
- }
+ post {
+ agent { label 'built-in' }
+ always {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ for (label in pullRequest.labels) {
+ if (label.contains("${Machine}")) {
+ pullRequest.removeLabel(label)
}
}
+ } catch (Exception e) {
+ echo "Failed to remove labels: ${e.getMessage()}"
}
}
- stage ('Success') {
- when {
- expression { return FINAL_STATE == 'PASSED' }
- }
- steps {
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Passed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add success label or comment: ${e.getMessage()}"
- }
- }
- }
+ }
+ }
+ success {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ pullRequest.addLabel("CI-${Machine}-Passed")
+ //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+ //sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add success label or comment: ${e.getMessage()}"
}
}
- stage('Failure') {
- when {
- expression { return FINAL_STATE == 'FAILED' }
- }
- steps {
- script {
- if(env.CHANGE_ID) {
- try {
- //pullRequest.addLabel("CI-${Machine}-Failed")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- //pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add failure label or comment: ${e.getMessage()}"
- }
- }
- }
+ }
+ }
+ failure {
+ script {
+ if(env.CHANGE_ID) {
+ try {
+ pullRequest.addLabel("CI-${Machine}-Failed")
+ //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
+ def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+ pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+ //sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
+ } catch (Exception e) {
+ echo "Failed to add failure label or comment: ${e.getMessage()}"
}
}
}
}
}
-
}
--
GitLab
From 95fdcf7bcdccf8e5ecad66851c0a240388c138a2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 20:43:26 +0000
Subject: [PATCH 097/189] removed agent on post
---
ci/Jenkinsfile | 1 -
1 file changed, 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index a77231130..5903f147a 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -142,7 +142,6 @@ pipeline {
stage('Run Tests') {
matrix {
- agent { label "${machine}-emc" }
axes {
axis {
name 'Case'
--
GitLab
From e8ce4e666ec5ecfb20d331e1bd00551a925c84d4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 20:49:04 +0000
Subject: [PATCH 098/189] removed agents again
---
ci/Jenkinsfile | 2 --
1 file changed, 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 5903f147a..7f29a015f 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -7,7 +7,6 @@ def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
pipeline {
- agent { label 'built-in' }
options {
skipDefaultCheckout()
@@ -221,7 +220,6 @@ pipeline {
}
post {
- agent { label 'built-in' }
always {
script {
if(env.CHANGE_ID) {
--
GitLab
From bb09e3f5384f75e0610f54e0dbabef2de763e250 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 20:53:05 +0000
Subject: [PATCH 099/189] main pipeline does need agent
---
ci/Jenkinsfile | 2 ++
1 file changed, 2 insertions(+)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 7f29a015f..366a42a29 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -8,6 +8,8 @@ def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
pipeline {
+ agent { label 'built-in' }
+
options {
skipDefaultCheckout()
parallelsAlwaysFailFast()
--
GitLab
From 8da39375312bed8e80366838fbb5432409acf70e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 21:06:23 +0000
Subject: [PATCH 100/189] accedetally removed agent from Run Tests
---
ci/Jenkinsfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 366a42a29..d2ec8dd67 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -143,6 +143,7 @@ pipeline {
stage('Run Tests') {
matrix {
+ agent { label "${machine}-emc" }
axes {
axis {
name 'Case'
--
GitLab
From 4fdb7daf89026214a0308ae278493620253ece07 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 21:26:08 +0000
Subject: [PATCH 101/189] chained fail critera to nead at least on DEAD and
scaping stat values bug fix by addding semi colon
---
ci/scripts/run-check_ci.sh | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index c593e73f6..ad619708c 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -78,8 +78,9 @@ while true; do
num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done | cut -d: -f2) || true
- num_succeeded=$(echo "${rocotostat_output}" | grep SUCCEEDED | cut -d: -f2) || true
- num_failed=$(echo "${rocotostat_output}" | grep FAIL | cut -d: -f2) || true
+ num_succeeded=$(echo "${rocotostat_output}" | grep "SUCCEEDED:" | cut -d: -f2) || true
+ num_failed=$(echo "${rocotostat_output}" | grep "FAIL:" | cut -d: -f2) || true
+ num_dead=$(echo "${rocotostat_output}" | grep "DEAD:" | cut -d: -f2) || true
rocoto_stat=$(echo "${rocotostat_output}" | tail -1) || true
echo "(${pslot}) Total Cycles: ${num_cycles} number done: ${num_cycles_done} ${rocoto_stat} on ${MACHINE_ID^}"
@@ -89,7 +90,7 @@ while true; do
echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed or dead at $(date)" || true
echo "Experiment ${pslot} Terminated: *${rocoto_stat}*"
} | tee -a "${run_check_logfile}"
- if [[ "${num_failed}" -ne 0 ]]; then
+ if [[ "${num_dead}" -ne 0 ]]; then
error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
{
echo "Error logs:"
--
GitLab
From bc4fa4c8e2a57e6654a8951e2ad32491bbc3d09b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 21:34:47 +0000
Subject: [PATCH 102/189] miss typed the work script when sending message to PR
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index d2ec8dd67..f7e46f0ce 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -203,7 +203,7 @@ pipeline {
repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim()
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
try {
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
--
GitLab
From a85ac3a89a9d457e846c1c099fc9ad6d1385c545 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 22:04:34 +0000
Subject: [PATCH 103/189] put publish script inside try block
---
ci/Jenkinsfile | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index f7e46f0ce..e163a5469 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -108,7 +108,7 @@ pipeline {
catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
}
}
- repo_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
try {
pullRequest.comment("Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
@@ -200,11 +200,10 @@ pipeline {
}
}
}
- repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim()
- gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
try {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+ gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
-
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
--
GitLab
From 4c921cf99a3e321e6e6cf845f108fa50dd790920 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 22:44:17 +0000
Subject: [PATCH 104/189] turned off failFast and made file hlding all the
errror logs unique to pslot name
---
ci/Jenkinsfile | 14 +++++++++-----
ci/scripts/run-check_ci.sh | 4 ++--
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index e163a5469..e302ef39b 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -173,6 +173,9 @@ pipeline {
when {
expression { return caseList.contains(Case) }
}
+ options {
+ failFast false
+ }
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
@@ -185,8 +188,9 @@ pipeline {
ws(HOME) {
def error_logs = ""
def error_logs_message = ""
- if (fileExists("RUNTESTS/error.logs")) {
- def fileContent = readFile 'RUNTESTS/error.logs'
+ def error_file = "RUNTESTS/${pslot}_error.logs"
+ if (fileExists(error_file)) {
+ def fileContent = readFile error_file
def lines = fileContent.readLines()
for (line in lines) {
echo "archiving: ${line}"
@@ -201,14 +205,14 @@ pipeline {
}
}
try {
- sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
- gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+ gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
} else {
- echo "No error logs found for failed cases in $HOME/RUNTESTS/error.logs"
+ echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
error("Failed to run experiments ${Case} on ${Machine}")
}
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index ad619708c..5ef860b67 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -96,9 +96,9 @@ while true; do
echo "Error logs:"
echo "${error_logs}"
} | tee -a "${run_check_logfile}"
- rm -f "${RUNTESTS}/error.logs"
+ rm -f "${RUNTESTS}/${pslot}_error.logs"
for log in ${error_logs}; do
- echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/error.logs"
+ echo "RUNTESTS${log#*RUNTESTS}" >> "${RUNTESTS}/${pslot}_error.logs"
done
fi
rc=1
--
GitLab
From ce1d12eb0e2b0a99ca835f21af010be4eb759a15 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 22:54:47 +0000
Subject: [PATCH 105/189] failFast false syntax
---
ci/Jenkinsfile | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index e302ef39b..c98edd6c3 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -173,9 +173,7 @@ pipeline {
when {
expression { return caseList.contains(Case) }
}
- options {
- failFast false
- }
+ failFast false
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
--
GitLab
From dc8db2fb8d57f097493a6c53cf80cace583a1f18 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 23:32:59 +0000
Subject: [PATCH 106/189] removed post and added a finilize stage that only
marks success and made fail inside matrix
---
ci/Jenkinsfile | 65 +++++++++++++-------------------------------------
1 file changed, 17 insertions(+), 48 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index c98edd6c3..1952088cf 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -5,6 +5,7 @@ def caseList = ''
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
+def STATUS = 'PASSED'
pipeline {
@@ -203,15 +204,20 @@ pipeline {
}
}
try {
- sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
} else {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
}
@@ -220,56 +226,19 @@ pipeline {
}
}
}
- }
- }
-
- post {
- always {
- script {
- if(env.CHANGE_ID) {
- try {
- //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- for (label in pullRequest.labels) {
- if (label.contains("${Machine}")) {
- pullRequest.removeLabel(label)
- }
+ stage( 'FINALIZE' ) {
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ if (STATUS == 'PASSED') {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
}
- } catch (Exception e) {
- echo "Failed to remove labels: ${e.getMessage()}"
}
}
- }
- }
- success {
- script {
- if(env.CHANGE_ID) {
- try {
- pullRequest.addLabel("CI-${Machine}-Passed")
- //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
- //sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add success label or comment: ${e.getMessage()}"
- }
- }
- }
- }
- failure {
- script {
- if(env.CHANGE_ID) {
- try {
- pullRequest.addLabel("CI-${Machine}-Failed")
- //sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """)
- def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
- pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
- //sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
- } catch (Exception e) {
- echo "Failed to add failure label or comment: ${e.getMessage()}"
- }
- }
- }
+ }
}
}
}
--
GitLab
From 6abfaa5aa12a96867bbe705f3a1b1d0859434f45 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 5 Apr 2024 23:39:17 +0000
Subject: [PATCH 107/189] Had new brakets wrong
---
ci/Jenkinsfile | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 1952088cf..54749c4ba 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -226,19 +226,19 @@ pipeline {
}
}
}
- stage( 'FINALIZE' ) {
- agent { label "${machine}-emc" }
- steps {
- script {
- if (STATUS == 'PASSED') {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
- }
+ }
+ stage( 'FINALIZE' ) {
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ if (STATUS == 'PASSED') {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
}
}
- }
- }
+ }
+ }
}
}
--
GitLab
From ce9666e2d05ecceaec54b62d53413795ed596eec Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 00:15:18 +0000
Subject: [PATCH 108/189] remoted timestamp and quote errrors
---
ci/Jenkinsfile | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 54749c4ba..291379ee6 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -109,10 +109,10 @@ pipeline {
catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
}
}
- sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
- gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
try {
- pullRequest.comment("Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})")
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+ gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url}")""")
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory \`${HOME}\`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
@@ -232,10 +232,10 @@ pipeline {
steps {
script {
if (STATUS == 'PASSED') {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at <br>Built and ran in directory \`${HOME}\`" """, returnStatus: true)
}
}
}
--
GitLab
From ee245eb000b6f31b165e12a98e48b717afafa452 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 00:18:10 +0000
Subject: [PATCH 109/189] removed escapes to tics on home
---
ci/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 291379ee6..61ee14007 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory \`${HOME}\`" """, returnStatus: true)
+ sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
@@ -235,7 +235,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at <br>Built and ran in directory \`${HOME}\`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
}
}
}
--
GitLab
From b28ccf40a5f1f56c468cfe5c06b66ed7332d2f44 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 00:45:05 +0000
Subject: [PATCH 110/189] misstyped script again and gave full path to error
log file
---
ci/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 61ee14007..d7a7ec994 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -187,7 +187,7 @@ pipeline {
ws(HOME) {
def error_logs = ""
def error_logs_message = ""
- def error_file = "RUNTESTS/${pslot}_error.logs"
+ def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
if (fileExists(error_file)) {
def fileContent = readFile error_file
def lines = fileContent.readLines()
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(sript: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
--
GitLab
From 6c8a0b49111170932564578b8713557e58927d8f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 01:06:13 +0000
Subject: [PATCH 111/189] groovy uses double back slash to escape trying to get
tic markdown in message to GitHug
---
ci/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index d7a7ec994..b93aefa78 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: \\`\\`\\`${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
--
GitLab
From 23c36de05d8c63eeba96586df924ce06bdf627ad Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 01:22:47 +0000
Subject: [PATCH 112/189] just some fine tunning on format
---
ci/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index b93aefa78..f6ae927b1 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: \\`\\`\\`${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: \\`\\`\\`\\\n${error_logs_message}\\\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "CI **FAILED** on ${Machine} at $(date +'%D %r')<br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
--
GitLab
From 64f6f3af7a8fa8e8883066c0032ff93ce277afdf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 01:25:57 +0000
Subject: [PATCH 113/189] just some fine tunning on format still
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index f6ae927b1..83543dac7 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -216,7 +216,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "CI **FAILED** on ${Machine} at $(date +'%D %r')<br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "CI **FAILED** on ${Machine}<br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
--
GitLab
From b970e398a28e7317e4438d87b66650be309bfc57 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 01:39:18 +0000
Subject: [PATCH 114/189] one more iteration on format
---
ci/Jenkinsfile | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 83543dac7..9a2d844aa 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs: \\`\\`\\`\\\n${error_logs_message}\\\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs:\\\n\\\n\\`\\`\\`\\\n${error_logs_message}\\\n\\`\\`\\`\\\n\\\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -213,10 +213,6 @@ pipeline {
} else {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "CI **FAILED** on ${Machine}<br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
STATUS = 'FAILED'
error("Failed to run experiments ${Case} on ${Machine}")
}
@@ -237,6 +233,12 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """, returnStatus: true)
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
}
+ else {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHCHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ }
}
}
}
--
GitLab
From a881648dd6d57027089077d9c1aad87aa449e304 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 02:11:41 +0000
Subject: [PATCH 115/189] move failFast not up one level
---
ci/Jenkinsfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 9a2d844aa..3c8aa553a 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -143,6 +143,7 @@ pipeline {
}
stage('Run Tests') {
+ failFast false
matrix {
agent { label "${machine}-emc" }
axes {
@@ -174,7 +175,7 @@ pipeline {
when {
expression { return caseList.contains(Case) }
}
- failFast false
+ failFast false
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
--
GitLab
From e84e5a0185e4f2a3039bf4b84256c12344f14876 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 6 Apr 2024 02:31:58 +0000
Subject: [PATCH 116/189] remove all fail fasts and forced error
---
ci/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 3c8aa553a..542ef000f 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -13,7 +13,7 @@ pipeline {
options {
skipDefaultCheckout()
- parallelsAlwaysFailFast()
+ //parallelsAlwaysFailFast()
}
stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
@@ -215,7 +215,7 @@ pipeline {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
STATUS = 'FAILED'
- error("Failed to run experiments ${Case} on ${Machine}")
+ echo "Failed to run experiments ${Case} on ${Machine}"
}
}
}
--
GitLab
From 2e9331f6c7a99d6ef7d85ae2e32f6c01c69b9e1b Mon Sep 17 00:00:00 2001
From: terry mcguinness
<terry.mcguinness@dlogin07.dogwood.wcoss2.ncep.noaa.gov>
Date: Wed, 10 Apr 2024 16:49:47 +0000
Subject: [PATCH 117/189] test commit permissions on dogwood
---
ci/scripts/utils/rocotostat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index b73ddc5f1..03e8ef8f8 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -11,7 +11,7 @@ logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=Fals
def input_args():
"""
- Parse command-line arguments.
+ Parse command-line arguments. test
Returns
-------
--
GitLab
From ae6d259936478624afbef10b0bc729a3b8e65fff Mon Sep 17 00:00:00 2001
From: terry mcguinness
<terry.mcguinness@dlogin07.dogwood.wcoss2.ncep.noaa.gov>
Date: Wed, 10 Apr 2024 16:51:09 +0000
Subject: [PATCH 118/189] confermend push auth on dogwood
---
ci/scripts/utils/rocotostat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index 03e8ef8f8..b73ddc5f1 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -11,7 +11,7 @@ logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=Fals
def input_args():
"""
- Parse command-line arguments. test
+ Parse command-line arguments.
Returns
-------
--
GitLab
From 6d3102f0d51179a687555adcf13bd75a6e970f03 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 15:51:00 +0000
Subject: [PATCH 119/189] disable shell check quote globbing for two lines
because the quotes needed to be removed for the command to work as intended
---
ci/scripts/check_ci.sh | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 54c17d63b..4d43492af 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -132,10 +132,12 @@ for pr in ${pr_list}; do
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
+ # shellcheck disable=SC2086
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")"
+ # shellcheck enable=SC2086
{
- echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')"
+ echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" || true
echo ""
echo "Error logs:"
echo "\`\`\`"
--
GitLab
From 776c4baf4de36754a7abde9603375a7b23ddfbe3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 16:33:38 +0000
Subject: [PATCH 120/189] added feature to export results for bash
---
ci/scripts/utils/rocotostat.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index b73ddc5f1..13ac7d47f 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -32,6 +32,7 @@ def input_args():
parser.add_argument('-d', help='database_file', metavar='Database File', type=FileType('r'), required=True)
parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
parser.add_argument('-v', action='store_true', help='List the states and the number of jobs that are in each', required=False)
+ parser.add_argument('--export', help='create and export list of the status values for bash', required=False)
args = parser.parse_args()
@@ -105,5 +106,10 @@ if __name__ == '__main__':
else:
print(f'Number of {status} : {rocoto_status[status]}')
- print(rocoto_state)
+ if args.export:
+ for status_type, status in rocoto_status.items():
+ print(f'export {status_type}={rocoto_status[status]}')
+ else:
+ print(rocoto_state)
+
sys.exit(error_return)
--
GitLab
From 3f02523b64dfce0e283ab305d27d9415e403722d Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 11:53:36 -0500
Subject: [PATCH 121/189] added and tested export feature to rocoto stauts tool
---
ci/scripts/utils/rocotostat.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index 13ac7d47f..59b24eebc 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -32,7 +32,7 @@ def input_args():
parser.add_argument('-d', help='database_file', metavar='Database File', type=FileType('r'), required=True)
parser.add_argument('--verbose', action='store_true', help='List the states and the number of jobs that are in each', required=False)
parser.add_argument('-v', action='store_true', help='List the states and the number of jobs that are in each', required=False)
- parser.add_argument('--export', help='create and export list of the status values for bash', required=False)
+ parser.add_argument('--export', action='store_true', help='create and export list of the status values for bash', required=False)
args = parser.parse_args()
@@ -66,8 +66,8 @@ def rocoto_statcount():
rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1]
rocoto_status = {
- 'Cycles': len(rocotostat_output),
- 'Cycles_Done': sum([sublist.count('Done') for sublist in rocotostat_output])
+ 'CYCLES_TOTAL': len(rocotostat_output),
+ 'CYCLES_DONE': sum([sublist.count('Done') for sublist in rocotostat_output])
}
status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED']
@@ -84,12 +84,12 @@ if __name__ == '__main__':
error_return = 0
rocoto_status = rocoto_statcount()
- if rocoto_status['Cycles'] == rocoto_status['Cycles_Done']:
- print(f"All {rocoto_status['Cycles']} Cycles are Done")
+ if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']:
+ print(f"All {rocoto_status['CYCLES']} Cycles are Done")
rocoto_state = 'DONE'
elif rocoto_status['DEAD'] > 0:
error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
- rocoto_state = 'FAILED or DEAD'
+ rocoto_state = 'FAIL'
elif 'UNKNOWN' in rocoto_status:
error_return = rocoto_status['UNKNOWN']
rocoto_state = 'UNKNOWN'
@@ -99,6 +99,8 @@ if __name__ == '__main__':
else:
rocoto_state = 'RUNNING'
+ rocoto_status['STATE'] = rocoto_state
+
if args.verbose or args.v:
for status in rocoto_status:
if args.v:
@@ -107,8 +109,8 @@ if __name__ == '__main__':
print(f'Number of {status} : {rocoto_status[status]}')
if args.export:
- for status_type, status in rocoto_status.items():
- print(f'export {status_type}={rocoto_status[status]}')
+ for status in rocoto_status:
+ print(f'export {status}={rocoto_status[status]}')
else:
print(rocoto_state)
--
GitLab
From 9f2e34ce108f7e298e8df0879987d4f48e0bc495 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:06:14 +0000
Subject: [PATCH 122/189] added export feature output for rocotostat in run
check script
---
ci/scripts/run-check_ci.sh | 23 ++++++++---------------
ci/scripts/utils/rocotostat.py | 2 +-
2 files changed, 9 insertions(+), 16 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 5ef860b67..59403cabd 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,24 +73,17 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
- rocotostat_output="$("${ROOT_DIR}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)"
+ eval "$("${ROOT_DIR}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)"
error_stat=$?
- num_cycles=$(echo "${rocotostat_output}" | grep "Cycles:" | cut -d: -f2 ) || true
- num_cycles_done=$(echo "${rocotostat_output}" | grep Cycles_Done | cut -d: -f2) || true
- num_succeeded=$(echo "${rocotostat_output}" | grep "SUCCEEDED:" | cut -d: -f2) || true
- num_failed=$(echo "${rocotostat_output}" | grep "FAIL:" | cut -d: -f2) || true
- num_dead=$(echo "${rocotostat_output}" | grep "DEAD:" | cut -d: -f2) || true
- rocoto_stat=$(echo "${rocotostat_output}" | tail -1) || true
-
- echo "(${pslot}) Total Cycles: ${num_cycles} number done: ${num_cycles_done} ${rocoto_stat} on ${MACHINE_ID^}"
+ echo "(${pslot}) Total Cycles: ${CYCLES_TOTAL} number done: ${CYCLES_DONE} ${ROCOTO_STATE} on ${MACHINE_ID^}"
if [[ ${error_stat} -ne 0 ]]; then
{
- echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed or dead at $(date)" || true
- echo "Experiment ${pslot} Terminated: *${rocoto_stat}*"
+ echo "Experiment ${pslot} Terminated with ${FAIL} tasks failed and ${DEAD} dead at $(date)" || true
+ echo "Experiment ${pslot} Terminated: *${ROCOTO_STATE}*"
} | tee -a "${run_check_logfile}"
- if [[ "${num_dead}" -ne 0 ]]; then
+ if [[ "${DEAD}" -ne 0 ]]; then
error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
{
echo "Error logs:"
@@ -105,10 +98,10 @@ while true; do
break
fi
- if [[ "${rocoto_stat}" == "DONE" ]]; then
+ if [[ "${ROCOTO_STAT}" == "DONE" ]]; then
{
- echo "Experiment ${pslot} Completed ${num_cycles_done} Cycles at $(date)" || true
- echo "with ${num_succeeded} successfully completed jobs" || true
+ echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles at $(date)" || true
+ echo "with ${SUCCEEDED} successfully completed jobs" || true
echo "Experiment ${pslot} Completed: *SUCCESS*"
} | tee -a "${run_check_logfile}"
rc=0
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index 59b24eebc..05d82cb41 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -99,7 +99,7 @@ if __name__ == '__main__':
else:
rocoto_state = 'RUNNING'
- rocoto_status['STATE'] = rocoto_state
+ rocoto_status['ROCOTO_STATE'] = rocoto_state
if args.verbose or args.v:
for status in rocoto_status:
--
GitLab
From d8d7798813474d7ec081b9e66daa5929bde4cb09 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:14:59 +0000
Subject: [PATCH 123/189] updated some docstring input vaules
---
ci/scripts/utils/pr_list_database.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index f6af7880a..849bc9535 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -90,9 +90,9 @@ def display_db(db, display) -> list:
Parameters
----------
- ci_database : SQLiteDB
+ db : SQLiteDB
The database to display.
- args : argparse.Namespace
+ args : display
The command line arguments.
Returns
@@ -121,7 +121,7 @@ def update_database(db: SQLiteDB) -> list:
Parameters
----------
- ci_database : SQLiteDB
+ db : SQLiteDB
The database to update.
Returns
--
GitLab
From 41e30645d720493664d1aa8fd2a624931ffc5ba0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:29:50 +0000
Subject: [PATCH 124/189] trying to remove SC2083 shellnorm complaints
---
ci/scripts/check_ci.sh | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 4d43492af..766f60021 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -132,10 +132,8 @@ for pr in ${pr_list}; do
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- # shellcheck disable=SC2086
- ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
- gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")"
- # shellcheck enable=SC2086
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs:+ $error_logs} --repo "PR_${pr}" > /dev/null
+ gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs:+ $error_logs} --gist "PR_${pr}")"
{
echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" || true
echo ""
--
GitLab
From 08a1dddfb6762ef2c129272e37658fb604449bc1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:37:33 +0000
Subject: [PATCH 125/189] trying to remove SC2086 shellnorm complaints when I
specificly do not want quote in command line
---
ci/scripts/check_ci.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 766f60021..b3c82aa9f 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -132,8 +132,10 @@ for pr in ${pr_list}; do
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
- ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs:+ $error_logs} --repo "PR_${pr}" > /dev/null
- gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs:+ $error_logs} --gist "PR_${pr}")"
+ # shellcheck disable=SC2086
+ ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
+ # shellcheck disable=SC2086
+ gist_url="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file ${error_logs} --gist "PR_${pr}")"
{
echo "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')" || true
echo ""
--
GitLab
From 8404249e6447885cb2d37b5df710b10ee4320d16 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:39:43 +0000
Subject: [PATCH 126/189] extra spaces removed in rocotostat
---
ci/scripts/utils/rocotostat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index 05d82cb41..88884319d 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -111,7 +111,7 @@ if __name__ == '__main__':
if args.export:
for status in rocoto_status:
print(f'export {status}={rocoto_status[status]}')
- else:
+ else:
print(rocoto_state)
sys.exit(error_return)
--
GitLab
From cd3c456d47977bb026f980f0fcd42550267c974e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:47:00 +0000
Subject: [PATCH 127/189] dispelling need for SC2312
---
ci/scripts/run-check_ci.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 59403cabd..a7176391c 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,6 +73,7 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
+ # shellcheck disable=SC2312 # We want to use the exit code of the command
eval "$("${ROOT_DIR}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)"
error_stat=$?
--
GitLab
From 80fca8646464c5cc7069c014d052532e06a0e987 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 20:30:51 +0000
Subject: [PATCH 128/189] remove ROOT from run check and added system build as
third optonal arg
---
ci/Jenkinsfile | 3 +--
ci/scripts/clone-build_ci.sh | 2 +-
ci/scripts/run-check_ci.sh | 9 ++-------
3 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 542ef000f..deb346df6 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -179,10 +179,9 @@ pipeline {
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
- env.SYSTEM_BUILD_DIR=system // used to set the system build directory in the run scripts
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
try {
- sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
+ sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
} catch (Exception error_experment) {
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
ws(HOME) {
diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
index 3cef7fc23..11bc43aac 100755
--- a/ci/scripts/clone-build_ci.sh
+++ b/ci/scripts/clone-build_ci.sh
@@ -48,7 +48,7 @@ git clone "${REPO_URL}"
cd global-workflow || exit 1
# checkout pull request
-"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules
+"${GH}" pr checkout "${PR}" --repo "${REPO_URL}" --recurse-submodules -j 4
HOMEgfs="${PWD}"
source "${HOMEgfs}/ush/detect_machine.sh"
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index a7176391c..d186f9474 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -9,8 +9,7 @@ set -eu
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
-ROOT_DIR=${3:-${ROOT_DIR:-?}} # Location of the root of the global-workflow repository
- # that runs CI functions that may be outside of PRs
+SYSTEM_BUILD_DIR=${3:-${SYSTEM_BUILD_DIR:-?}} # Name of the system build directory
# TEST_DIR contains 2 directories;
# 1. HOMEgfs: clone of the global-workflow
@@ -29,10 +28,6 @@ HOMEgfs="${TEST_DIR}/${SYSTEM_BUILD_DIR:-global-workflow}"
RUNTESTS="${TEST_DIR}/RUNTESTS"
run_check_logfile="${RUNTESTS}/ci-run_check.log"
-if [[ "${ROOT_DIR}" == "?" ]]; then
- ROOT_DIR="${HOMEgfs}"
-fi
-
# Source modules and setup logging
echo "Source modules."
source "${HOMEgfs}/workflow/gw_setup.sh"
@@ -74,7 +69,7 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
# shellcheck disable=SC2312 # We want to use the exit code of the command
- eval "$("${ROOT_DIR}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)"
+ eval "$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)"
error_stat=$?
echo "(${pslot}) Total Cycles: ${CYCLES_TOTAL} number done: ${CYCLES_DONE} ${ROCOTO_STATE} on ${MACHINE_ID^}"
--
GitLab
From 7a7a5445baa6a2346360e5ff8b953b5a0676786f Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 17:58:49 -0500
Subject: [PATCH 129/189] fixed bug with eval on export feature from
rocotocheck.py
---
ci/scripts/run-check_ci.sh | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index d186f9474..fbe503c93 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -49,7 +49,7 @@ fi
# Launch experiment
echo "Launch experiment with Rocoto."
rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
-sleep 30
+sleep 10
if [[ ! -f "${db}" ]]; then
echo "FATAL ERROR: Rocoto database file ${db} not found, experiment ${pslot} failed, ABORT!"
exit 2
@@ -64,15 +64,16 @@ while true; do
rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
# Wait before running rocotostat
- sleep 30
+ sleep 10
# Get job statistics
echo "Gather Rocoto statistics"
# shellcheck disable=SC2312 # We want to use the exit code of the command
- eval "$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)"
+ eval=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)
error_stat=$?
+ eval "${eval}"
- echo "(${pslot}) Total Cycles: ${CYCLES_TOTAL} number done: ${CYCLES_DONE} ${ROCOTO_STATE} on ${MACHINE_ID^}"
+ echo -e "(${pslot} on ${MACHIE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
if [[ ${error_stat} -ne 0 ]]; then
{
--
GitLab
From ae08b07d13be4f3bef7326669b59517a75c839b4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 00:56:12 +0000
Subject: [PATCH 130/189] needed to update keyword CYCLES to CYCLES_TOTAL and
consolidated Final in Jenkins
---
ci/Jenkinsfile | 20 ++++++++------------
ci/scripts/utils/rocotostat.py | 2 +-
2 files changed, 9 insertions(+), 13 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index deb346df6..0cfdd7e50 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -5,7 +5,7 @@ def caseList = ''
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
-def STATUS = 'PASSED'
+def STATUS = 'Passed'
pipeline {
@@ -120,7 +120,7 @@ pipeline {
}
}
sh(script: './link_workflow.sh')
- // sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
}
}
if (env.CHANGE_ID && system == 'gfs') {
@@ -213,7 +213,7 @@ pipeline {
} else {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
- STATUS = 'FAILED'
+ STATUS = 'Failed'
echo "Failed to run experiments ${Case} on ${Machine}"
}
}
@@ -227,17 +227,13 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
- if (STATUS == 'PASSED') {
+ try {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Passed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI PASSED** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
- }
- else {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHCHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Failed" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI FAILED** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
}
}
}
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index 88884319d..c0b6a3bdd 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -85,7 +85,7 @@ if __name__ == '__main__':
rocoto_status = rocoto_statcount()
if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']:
- print(f"All {rocoto_status['CYCLES']} Cycles are Done")
+ print(f"All {rocoto_status['CYCLES_TOTAL']} Cycles are Done")
rocoto_state = 'DONE'
elif rocoto_status['DEAD'] > 0:
error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
--
GitLab
From 9a277f09d9a5924c839a9940d7c2b57f42a3ba08 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 01:03:03 +0000
Subject: [PATCH 131/189] added error after case fail catch
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 0cfdd7e50..6500c2c17 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -214,7 +214,7 @@ pipeline {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
STATUS = 'Failed'
- echo "Failed to run experiments ${Case} on ${Machine}"
+ error("Failed to run experiments ${Case} on ${Machine}")
}
}
}
--
GitLab
From a641b548e58ad2bf6ddbb73e9defc641e9813930 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 20:16:45 -0500
Subject: [PATCH 132/189] suppress All Done message when export is used with
rocotocheck
---
ci/scripts/utils/rocotostat.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index c0b6a3bdd..730def1d2 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -85,7 +85,8 @@ if __name__ == '__main__':
rocoto_status = rocoto_statcount()
if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']:
- print(f"All {rocoto_status['CYCLES_TOTAL']} Cycles are Done")
+ if not args.export:
+ print(f"All {rocoto_status['CYCLES_TOTAL']} Cycles are Done")
rocoto_state = 'DONE'
elif rocoto_status['DEAD'] > 0:
error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
--
GitLab
From d35b115b4503b9861100d208765aa65a84b8a8cb Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 12 Apr 2024 21:04:17 -0500
Subject: [PATCH 133/189] fixed couple of more bugs STAT vs STATE for one
---
ci/scripts/run-check_ci.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index fbe503c93..6f73d3080 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,7 +73,7 @@ while true; do
error_stat=$?
eval "${eval}"
- echo -e "(${pslot} on ${MACHIE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
+ echo -e "(${pslot} on ${MACHINE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
if [[ ${error_stat} -ne 0 ]]; then
{
@@ -95,7 +95,7 @@ while true; do
break
fi
- if [[ "${ROCOTO_STAT}" == "DONE" ]]; then
+ if [[ "${ROCOTO_STATE}" == "DONE" ]]; then
{
echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles at $(date)" || true
echo "with ${SUCCEEDED} successfully completed jobs" || true
--
GitLab
From 86096da2c611ddaa76a499ab5ab7b81993a1a967 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 02:20:12 +0000
Subject: [PATCH 134/189] fixing line breaks in error messages to gitHub
---
ci/Jenkinsfile | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 6500c2c17..7d93f1b80 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -112,7 +112,7 @@ pipeline {
try {
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} failed on ${Machine} with error logs:\\\n\\\n\\`\\`\\`\\\n${error_logs_message}\\\n\\`\\`\\`\\\n\\\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} *FAILED* on ${Machine} with error logs:\n\\`\\`\\`\\\n${error_logs_message}\n\\`\\`\\`\\\n\\\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -214,6 +214,14 @@ pipeline {
echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
}
STATUS = 'Failed'
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
error("Failed to run experiments ${Case} on ${Machine}")
}
}
--
GitLab
From 714057a0af124e74ddf5b7c3392c7780f2ab08c4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 02:32:32 +0000
Subject: [PATCH 135/189] Put when on finalize and put retruns in proper place
on error messatge
---
ci/Jenkinsfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 7d93f1b80..74bc0af02 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} *FAILED* on ${Machine} with error logs:\n\\`\\`\\`\\\n${error_logs_message}\n\\`\\`\\`\\\n\\\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -232,6 +232,7 @@ pipeline {
}
}
stage( 'FINALIZE' ) {
+ when( STATUS == 'Passed' )
agent { label "${machine}-emc" }
steps {
script {
--
GitLab
From 0f1a08f7e22c4db83bcea94cd80231f2796aebdb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 02:38:17 +0000
Subject: [PATCH 136/189] when is in brackets
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 74bc0af02..9e0828e30 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -232,7 +232,7 @@ pipeline {
}
}
stage( 'FINALIZE' ) {
- when( STATUS == 'Passed' )
+ when { STATUS == 'Passed' }
agent { label "${machine}-emc" }
steps {
script {
--
GitLab
From 294f060e780ff0eec5ee561fc53292cba8534575 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 02:42:49 +0000
Subject: [PATCH 137/189] when needs to be told it is an expression
---
ci/Jenkinsfile | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 9e0828e30..eb5233913 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -232,7 +232,11 @@ pipeline {
}
}
stage( 'FINALIZE' ) {
- when { STATUS == 'Passed' }
+ when {
+ expression {
+ STATUS == 'Passed'
+ }
+ }
agent { label "${machine}-emc" }
steps {
script {
--
GitLab
From c545e1cb2251de28da9fe441eeabd2ed7a996020 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 13 Apr 2024 03:47:12 +0000
Subject: [PATCH 138/189] exta log space and unnecessary comment on each fail
and sence run to completion just kill case batch jobs
---
ci/Jenkinsfile | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index eb5233913..23285ee34 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -183,7 +183,7 @@ pipeline {
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
} catch (Exception error_experment) {
- sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
ws(HOME) {
def error_logs = ""
def error_logs_message = ""
@@ -205,7 +205,7 @@ pipeline {
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -216,9 +216,7 @@ pipeline {
STATUS = 'Failed'
try {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
} catch (Exception e) {
echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
}
--
GitLab
From 342e2b8f6b272c01e908937f8ff02e66aae2597d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 13:07:04 +0000
Subject: [PATCH 139/189] added excapes on tic for HOME in pr message by groovy
protocals on Jenkins shell line
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 23285ee34..d21d1b878 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -242,7 +242,7 @@ pipeline {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory `${HOME}`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory //`${HOME}//`" """, returnStatus: true)
} catch (Exception e) {
echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
}
--
GitLab
From 115a5c6f550dd21256416e11cb1f6b98b9e6d856 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 13:12:59 +0000
Subject: [PATCH 140/189] update URLs back to global repo after development
tests
---
ci/scripts/check_ci.sh | 3 +--
ci/scripts/driver.sh | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index b3c82aa9f..cdd4e9f42 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -14,8 +14,7 @@ echo "Begin ${scriptname} at $(date -u)" || true
export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
GH=${HOME}/bin/gh
-#REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
-REPO_URL="git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git"
+REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
#########################################################################
# Set up runtime environment varibles for accounts on supproted machines
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index a94c72fac..3bedb379e 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -18,7 +18,7 @@ set -eux
# TODO using static build for GitHub CLI until fixed in HPC-Stack
#################################################################
export GH=${HOME}/bin/gh
-export REPO_URL=git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git
+export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
################################################################
# Setup the reletive paths to scripts and PS4 for better logging
--
GitLab
From 0cd8c623b4e32439c707d5d696810a2fbdde863a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 13:20:03 +0000
Subject: [PATCH 141/189] update URL in Jenkins file for Agent based PR updates
to use gh directly
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index d21d1b878..b5d3bdf07 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -4,7 +4,7 @@ def HOME = 'none'
def caseList = ''
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
-def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
+def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
def STATUS = 'Passed'
pipeline {
--
GitLab
From bfae583526308fefd829e205cba35dfebf9ad961 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 17:56:14 +0000
Subject: [PATCH 142/189] added back bash scraping wiht -v switch to
rocotostat.py
---
ci/scripts/run-check_ci.sh | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 6f73d3080..4a9ff57d8 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -69,9 +69,15 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
# shellcheck disable=SC2312 # We want to use the exit code of the command
- eval=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" --export)
+ eval=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
error_stat=$?
- eval "${eval}"
+
+ CYCLES_TOTAL=$(echo "${eval}" | grep CYCLES_TOTAL | cut -d: -f2) || true
+ CYCLES_DONE=$(echo "${eval}" | grep CYCLES_DONE | cut -d: -f2) || true
+ SUCCEEDED=$(echo "${eval}" | grep SUCCEEDED | cut -d: -f2) || true
+ FAIL=$(echo "${eval}" | grep FAIL | cut -d: -f2) || true
+ DEAD=$(echo "${eval}" | grep DEAD | cut -d: -f2) || true
+ ROCOTO_STATE=$(echo "${eval}" | tail -1) || true
echo -e "(${pslot} on ${MACHINE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
--
GitLab
From 70b169ec59b4fcef41909c4c770df407bd43e01f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Tue, 16 Apr 2024 13:57:43 -0400
Subject: [PATCH 143/189] Update ci/Jenkinsfile
sure can
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
ci/Jenkinsfile | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index b5d3bdf07..223c1237b 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -49,8 +49,7 @@ pipeline {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Ready" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
}
echo "Building and running on ${Machine} in directory ${HOME}"
}
--
GitLab
From 8def91bfbc8b318b748d99a66ded71ff90865053 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 19:19:03 +0000
Subject: [PATCH 144/189] updated option thired argument to run-check_ci.sh to
default to global-workflow as the system build dir name
---
ci/scripts/run-check_ci.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 4a9ff57d8..8f1ee1b17 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -9,7 +9,7 @@ set -eu
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
pslot=${2:-${pslot:-?}} # Name of the experiment being tested by this script
-SYSTEM_BUILD_DIR=${3:-${SYSTEM_BUILD_DIR:-?}} # Name of the system build directory
+SYSTEM_BUILD_DIR=${3:-"global-workflow"} # Name of the system build directory, default is "global-workflow
# TEST_DIR contains 2 directories;
# 1. HOMEgfs: clone of the global-workflow
@@ -24,7 +24,7 @@ SYSTEM_BUILD_DIR=${3:-${SYSTEM_BUILD_DIR:-?}} # Name of the system build direct
# └── ${pslot}
# Two system build directories created at build time gfs, and gdas
# TODO: Make this configurable (for now all scripts run from gfs for CI at runtime)
-HOMEgfs="${TEST_DIR}/${SYSTEM_BUILD_DIR:-global-workflow}"
+HOMEgfs="${TEST_DIR}/${SYSTEM_BUILD_DIR}"
RUNTESTS="${TEST_DIR}/RUNTESTS"
run_check_logfile="${RUNTESTS}/ci-run_check.log"
--
GitLab
From 0c55ed2ec59cbc9c754e7a9a03af4758f81cc95d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 19:29:02 +0000
Subject: [PATCH 145/189] Consoldated label update lines in Jenkins
---
ci/Jenkinsfile | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 223c1237b..fe767dd4d 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -124,8 +124,7 @@ pipeline {
}
if (env.CHANGE_ID && system == 'gfs') {
try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" """)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
} catch (Exception e) {
echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
}
@@ -214,8 +213,7 @@ pipeline {
}
STATUS = 'Failed'
try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
} catch (Exception e) {
echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
}
@@ -238,9 +236,7 @@ pipeline {
steps {
script {
try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Building" """, returnStatus: true)
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory //`${HOME}//`" """, returnStatus: true)
} catch (Exception e) {
echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
--
GitLab
From 76d11986ab465363421b3ac14e27015b9b0db2d5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 20:27:17 +0000
Subject: [PATCH 146/189] skipped error logs with case is stalled and update a
few review comments
---
ci/scripts/check_ci.sh | 6 ++++++
ci/scripts/run-check_ci.sh | 2 +-
ci/scripts/utils/pr_list_database.py | 4 ++--
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index f596aab42..133c9ae2d 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -140,6 +140,12 @@ for pr in ${pr_list}; do
rm -f "${output_ci_single}"
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
+ if [[ "${rocoto_state}" == "STALLED" ]]; then
+ "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')"
+ "${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+ cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
+ exit "${rocoto_error}"
+ fi
error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
# shellcheck disable=SC2086
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo "PR_${pr}" > /dev/null
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 8f1ee1b17..764d13f4c 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -4,7 +4,7 @@ set -eu
#####################################################################################
# Script description: script to check the status of an experiment as reported
-# by Rocoto
+# by Rocoto
#####################################################################################
TEST_DIR=${1:-${TEST_DIR:-?}} # Location of the root of the testing directory
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index 849bc9535..fd396d004 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -84,7 +84,7 @@ def update_pr(db: SQLiteDB, args):
db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display_db(db, display) -> list:
+def display_db(db: SQLiteDB, display: any) -> list:
"""
Display the database.
@@ -92,7 +92,7 @@ def display_db(db, display) -> list:
----------
db : SQLiteDB
The database to display.
- args : display
+ display : argparse.Namespace
The command line arguments.
Returns
--
GitLab
From 147bab35260d016eb409e41843a7b5af9ce6e481 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 20:36:22 +0000
Subject: [PATCH 147/189] added spacers after doctrings for readablity
---
ci/scripts/utils/pr_list_database.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index fd396d004..95752ed37 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -39,6 +39,7 @@ def create_table(db: SQLiteDB):
db : SQLiteDB
The database to create.
"""
+
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])
@@ -53,6 +54,7 @@ def add_pr(db: SQLiteDB, pr: str) -> bool:
pr : str
The pull request to add.
"""
+
entities = (pr, 'Open', 'Ready', 0, 'ci_repo')
try:
db.insert_data('pr_list', entities)
@@ -74,6 +76,7 @@ def update_pr(db: SQLiteDB, args):
args : argparse.Namespace
The command line arguments.
"""
+
if len(args.update_pr) < 2:
print(f"update_pr must have at least one vaule to update")
sys.exit(0)
@@ -100,6 +103,7 @@ def display_db(db: SQLiteDB, display: any) -> list:
list
The rows of the database.
"""
+
values = []
if len(display) == 1:
rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
@@ -129,6 +133,7 @@ def update_database(db: SQLiteDB) -> list:
list
The kill list of pull requests.
"""
+
gh = GitHubPR()
pr_ready_list, pr_kill_list = gh.get_open_pr_list()
for pr in pr_ready_list:
@@ -148,9 +153,9 @@ def input_args():
argparse.Namespace
The parsed command line arguments.
"""
-
description = """Arguments for creating and updating db file for pr states
"""
+
parser = ArgumentParser(description=description,
formatter_class=ArgumentDefaultsHelpFormatter)
--
GitLab
From 5231e188d241bf5cc28ff8f74792a5af3c578d22 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 15:01:35 +0000
Subject: [PATCH 148/189] docstring indent for function
---
ci/scripts/utils/ci_utils.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index c88bf608a..fac3798c9 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -128,10 +128,10 @@ function create_experiment () {
}
function publish_logs() {
-# publish_logs function
-# This function takes a directory path and a list of files as arguments.
-# It calls the publish_logs.py script to publish the logs and returns its gist URL.
-# Usage: publish_logs <ID> <dir_path> <file1> <file2> ... <fileN>
+ # publish_logs function
+ # This function takes a directory path and a list of files as arguments.
+ # It calls the publish_logs.py script to publish the logs and returns its gist URL.
+ # Usage: publish_logs <ID> <dir_path> <file1> <file2> ... <fileN>
local PR_header="$1"
local dir_path="$2"
local file="$3"
--
GitLab
From b1a94b6673e44bcb5ed57532716195a4b87429d5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 15:07:59 +0000
Subject: [PATCH 149/189] put back consistent naming for ci database
---
ci/scripts/utils/pr_list_database.py | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index 95752ed37..e618ce31c 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -43,7 +43,7 @@ def create_table(db: SQLiteDB):
db.create_table('pr_list', ['pr INTEGER PRIMARY KEY UNIQUE', 'state TEXT', 'status TEXT', 'reset_id INTEGER', 'cases TEXT'])
-def add_pr(db: SQLiteDB, pr: str) -> bool:
+def add_pr(ci_database: SQLiteDB, pr: str) -> bool:
"""
Add a pull request to the database.
@@ -57,7 +57,7 @@ def add_pr(db: SQLiteDB, pr: str) -> bool:
entities = (pr, 'Open', 'Ready', 0, 'ci_repo')
try:
- db.insert_data('pr_list', entities)
+ ci_database.insert_data('pr_list', entities)
return True
except (SQLiteDBError.IntegrityError) as e:
if 'unique' in str(e).lower():
@@ -65,7 +65,7 @@ def add_pr(db: SQLiteDB, pr: str) -> bool:
return False
-def update_pr(db: SQLiteDB, args):
+def update_pr(ci_database: SQLiteDB, args):
"""
Update a pull request in the database.
@@ -84,10 +84,10 @@ def update_pr(db: SQLiteDB, args):
update_list = ['state', 'status', 'reset_id', 'cases']
for value in args.update_pr[1:]:
update = update_list.pop(0)
- db.update_data('pr_list', update, value, 'pr', args.update_pr[0])
+ ci_database.update_data('pr_list', update, value, 'pr', args.update_pr[0])
-def display_db(db: SQLiteDB, display: any) -> list:
+def display_db(ci_database: SQLiteDB, display: any) -> list:
"""
Display the database.
@@ -106,18 +106,18 @@ def display_db(db: SQLiteDB, display: any) -> list:
values = []
if len(display) == 1:
- rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
+ rows = ci_database.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'], f"pr = '{display[0]}'")
if len(display) == 2:
- rows = db.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
+ rows = ci_database.fetch_data('pr_list', ['pr'], f"state = '{display[0]}' AND status = '{display[1]}'")
if len(display) == 0:
- rows = db.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
+ rows = ci_database.fetch_data('pr_list', ['pr', 'state', 'status', 'reset_id', 'cases'])
for row in rows:
values.append(' '.join(map(str, row)))
return values
-def update_database(db: SQLiteDB) -> list:
+def update_database(ci_database: SQLiteDB) -> list:
"""
Update the database from the GitHub PRs
- only PRs from host machine are added to the database
@@ -125,7 +125,7 @@ def update_database(db: SQLiteDB) -> list:
Parameters
----------
- db : SQLiteDB
+ ci_database : SQLiteDB
The database to update.
Returns
@@ -137,7 +137,7 @@ def update_database(db: SQLiteDB) -> list:
gh = GitHubPR()
pr_ready_list, pr_kill_list = gh.get_open_pr_list()
for pr in pr_ready_list:
- if not add_pr(db, str(pr)):
+ if not add_pr(ci_database, str(pr)):
if pr not in pr_kill_list:
pr_kill_list.append(pr)
pr_kill_list = list(set(pr_kill_list))
--
GitLab
From 6b2360af200e714bdc79bfe74886be24a55b3b1f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 15:15:18 +0000
Subject: [PATCH 150/189] disable shellcheck for using Wed Apr 17 15:15:18 UTC
2024
---
ci/scripts/check_ci.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 133c9ae2d..cd907d34a 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -141,6 +141,7 @@ for pr in ${pr_list}; do
if [[ "${rocoto_error}" -ne 0 ]]; then
"${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
if [[ "${rocoto_state}" == "STALLED" ]]; then
+ # shellcheck disable=SC2312
"${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body "Experiment ${pslot} **${rocoto_state}** on ${MACHINE_ID^} at $(date +'%D %r')"
"${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
cancel_all_batch_jobs "${pr_dir}/RUNTESTS"
--
GitLab
From 8b9918d9b1bfdba534219732978a3f62c94e694f Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 17 Apr 2024 18:04:12 -0400
Subject: [PATCH 151/189] Update ci/scripts/run-check_ci.sh
Nice
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
ci/scripts/run-check_ci.sh | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 764d13f4c..06c94cd80 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -72,12 +72,10 @@ while true; do
eval=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
error_stat=$?
- CYCLES_TOTAL=$(echo "${eval}" | grep CYCLES_TOTAL | cut -d: -f2) || true
- CYCLES_DONE=$(echo "${eval}" | grep CYCLES_DONE | cut -d: -f2) || true
- SUCCEEDED=$(echo "${eval}" | grep SUCCEEDED | cut -d: -f2) || true
- FAIL=$(echo "${eval}" | grep FAIL | cut -d: -f2) || true
- DEAD=$(echo "${eval}" | grep DEAD | cut -d: -f2) || true
- ROCOTO_STATE=$(echo "${eval}" | tail -1) || true
+ for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
+ declare ${!state}=$(echo "${full_state}" | grep "${state}" | cut -d: -f2) || true
+ done
+ ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
echo -e "(${pslot} on ${MACHINE_ID^})\n\tTotal Cycles: ${CYCLES_TOTAL}\n\tNumber Cycles done: ${CYCLES_DONE}\n\tState: ${ROCOTO_STATE}"
--
GitLab
From 0e01cb5ae22d5f8b4f22bf293b69bb80069553ab Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 17 Apr 2024 18:50:15 -0400
Subject: [PATCH 152/189] Update ci/scripts/run-check_ci.sh
code cleaning looks cleaner
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 06c94cd80..8044bb77c 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -69,7 +69,7 @@ while true; do
# Get job statistics
echo "Gather Rocoto statistics"
# shellcheck disable=SC2312 # We want to use the exit code of the command
- eval=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
+ full_state=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v)
error_stat=$?
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
--
GitLab
From 5bd001db4e0c06fa8ed63407fe068147d97707f1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 22:55:51 +0000
Subject: [PATCH 153/189] removing bang in declare from an eralier version
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index 8044bb77c..ffbf2eed3 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,7 +73,7 @@ while true; do
error_stat=$?
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
- declare ${!state}=$(echo "${full_state}" | grep "${state}" | cut -d: -f2) || true
+ declare ${state}=$(echo "${full_state}" | grep "${state}" | cut -d: -f2) || true
done
ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
--
GitLab
From 8b97d200709c3f87b918b43fc151cf8be9daa101 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 23:00:43 +0000
Subject: [PATCH 154/189] shellnorms complaining about needing quotes for not
splitting and added them to the evaluation echoing and scraping of the
full_state varibable
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index ffbf2eed3..cdc107981 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,7 +73,7 @@ while true; do
error_stat=$?
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
- declare ${state}=$(echo "${full_state}" | grep "${state}" | cut -d: -f2) || true
+ declare ${state}="$(echo "${full_state}" | grep "${state}" | cut -d: -f2)" || true
done
ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
--
GitLab
From 9b5f7dc282575a314418dc48deab4d5be68b86dc Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 17 Apr 2024 23:03:49 +0000
Subject: [PATCH 155/189] appease on more shellnorm for quoting
---
ci/scripts/run-check_ci.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
index cdc107981..ee089dadc 100755
--- a/ci/scripts/run-check_ci.sh
+++ b/ci/scripts/run-check_ci.sh
@@ -73,7 +73,7 @@ while true; do
error_stat=$?
for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do
- declare ${state}="$(echo "${full_state}" | grep "${state}" | cut -d: -f2)" || true
+ declare "${state}"="$(echo "${full_state}" | grep "${state}" | cut -d: -f2)" || true
done
ROCOTO_STATE=$(echo "${full_state}" | tail -1) || exit 1
--
GitLab
From c338a16c38e3c581241b3fb0e37e1fd51e9034a6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 02:26:42 +0000
Subject: [PATCH 156/189] spacers add to docstrings in pr list database
---
ci/scripts/utils/pr_list_database.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/ci/scripts/utils/pr_list_database.py b/ci/scripts/utils/pr_list_database.py
index e618ce31c..357961481 100755
--- a/ci/scripts/utils/pr_list_database.py
+++ b/ci/scripts/utils/pr_list_database.py
@@ -14,10 +14,12 @@ def full_path(string):
----------
string : str
The relative path of the file or directory.
+
Returns
-------
str
The absolute path of the file or directory.
+
Raises
------
NotADirectoryError
@@ -51,6 +53,7 @@ def add_pr(ci_database: SQLiteDB, pr: str) -> bool:
----------
ci_database : SQLiteDB
The database to add the pull request to.
+
pr : str
The pull request to add.
"""
@@ -73,6 +76,7 @@ def update_pr(ci_database: SQLiteDB, args):
----------
ci_database : SQLiteDB
The database to update the pull request in.
+
args : argparse.Namespace
The command line arguments.
"""
@@ -95,6 +99,7 @@ def display_db(ci_database: SQLiteDB, display: any) -> list:
----------
db : SQLiteDB
The database to display.
+
display : argparse.Namespace
The command line arguments.
--
GitLab
From 64c9e51a0322d9447ac2beedaf723751a860a409 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 13:53:28 +0000
Subject: [PATCH 157/189] had escape char going in the wrong direction in
message to gitHub PR
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index fe767dd4d..e1230861a 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -237,7 +237,7 @@ pipeline {
script {
try {
sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory //`${HOME}//`" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
} catch (Exception e) {
echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
}
--
GitLab
From 506ce82357474e85dc0a0a6b3b9e9d5ce0ab72db Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 14:59:23 +0000
Subject: [PATCH 158/189] had to demote pslot to be local because it was
crating run conditions in parallel
---
ci/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index e1230861a..9c5ef1ee9 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -177,7 +177,7 @@ pipeline {
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
- pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+ def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
} catch (Exception error_experment) {
--
GitLab
From 3c6a275c0a2b0f4b6477a1f7f3a260a7f11b5a37 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 19:34:47 +0000
Subject: [PATCH 159/189] trying to parameterize job runs in Jenkins
---
ci/Jenkinsfile_multilabel | 257 ++++++++++++++++++++++++++++++++++++++
1 file changed, 257 insertions(+)
create mode 100644 ci/Jenkinsfile_multilabel
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
new file mode 100644
index 000000000..ff66db85f
--- /dev/null
+++ b/ci/Jenkinsfile_multilabel
@@ -0,0 +1,257 @@
+def Machine = 'none'
+def machine = 'none'
+def HOME = 'none'
+def caseList = ''
+// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
+def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
+def STATUS = 'Passed'
+
+properties([
+ parameters([[
+ $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'machine', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ ])
+])
+
+pipeline {
+
+ agent { label 'built-in' }
+
+ options {
+ skipDefaultCheckout()
+ //parallelsAlwaysFailFast()
+ }
+
+ stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
+ // which is used to designate the Nodes in the Jenkins Controler by the agent label
+ // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+
+ stage('Get Machine') {
+ agent { label 'built-in' }
+ steps {
+ script {
+ // generateBaselineNode = []
+ // . for (label in pullRequest.labels) {
+ // listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
+ // node -> node.getLabelString().contains(label) ? node.name : null
+
+ // if ((label.matches(node.getLabelString()+"-(.*)"))) {
+ // matchedNode += node.getLabelString()
+ // }
+
+ // if ((label.matches(node.getLabelString()+"(.*)-emc"))) {
+ // generateBaselineNode += node.getLabelString()
+ // }
+ // }
+ // }
+ build(job: "EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ machine = hera
+ Machine = machine[0].toUpperCase() + machine.substring(1)
+ }
+ }
+ }
+
+ stage('Get Common Workspace') {
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
+ HOME = "${WORKSPACE}"
+ sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
+ }
+ echo "Building and running on ${Machine} in directory ${HOME}"
+ }
+ }
+ }
+
+ stage('Build System') {
+ matrix {
+ agent { label "${machine}-emc" }
+ //options {
+ // throttle(['global_matrix_build'])
+ //}
+ axes {
+ axis {
+ name 'system'
+ values 'gfs', 'gefs'
+ }
+ }
+ stages {
+ stage('build system') {
+ steps {
+ script {
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
+ sh(script: "mkdir -p ${HOMEgfs}")
+ ws(HOMEgfs) {
+ if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
+ sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
+ checkout scm
+ dir('sorc') {
+ sh(script: './link_workflow.sh')
+ }
+ } else {
+ checkout scm
+ def gist_url = ""
+ def error_logs = ""
+ def error_logs_message = ""
+ def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
+ def build_args_list = builds_file['builds']
+ def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
+ dir("${HOMEgfs}/sorc") {
+ try {
+ sh(script: "${build_args}")
+ } catch (Exception error_build) {
+ echo "Failed to build system: ${error_build.getMessage()}"
+ if ( fileExists("logs/error.logs") ) {
+ def fileContent = readFile 'logs/error.logs'
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${line}") && readFile("${line}").length() > 0 ){
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
+ error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
+ }
+ catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
+ }
+ }
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+ gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ error("Failed to build system on ${Machine}")
+ }
+ }
+ sh(script: './link_workflow.sh')
+ sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ }
+ }
+ if (env.CHANGE_ID && system == 'gfs') {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
+ } catch (Exception e) {
+ echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+ }
+ }
+ if (system == 'gfs') {
+ caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ stage('Run Tests') {
+ failFast false
+ matrix {
+ agent { label "${machine}-emc" }
+ axes {
+ axis {
+ name 'Case'
+ // TODO add dynamic list of cases from env vars (needs addtional plugins)
+ values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
+ }
+ }
+ stages {
+
+ stage('Create Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ steps {
+ script {
+ sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
+ def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
+ system = yaml_case.experiment.system
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
+ env.RUNTESTS = "${HOME}/RUNTESTS"
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml")
+ }
+ }
+ }
+
+ stage('Run Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ failFast false
+ steps {
+ script {
+ HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
+ def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
+ } catch (Exception error_experment) {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
+ ws(HOME) {
+ def error_logs = ""
+ def error_logs_message = ""
+ def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
+ if (fileExists(error_file)) {
+ def fileContent = readFile error_file
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${HOME}/${line}") && readFile("${HOME}/${line}").length() > 0) {
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOME}/${line} "
+ error_logs_message = error_logs_message + "${HOME}/${line}\n"
+ } catch (Exception error_arch) {
+ echo "Failed to archive error log ${line}: ${error_arch.getMessage()}"
+ }
+ }
+ }
+ try {
+ gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ } else {
+ echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
+ }
+ STATUS = 'Failed'
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ error("Failed to run experiments ${Case} on ${Machine}")
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ stage( 'FINALIZE' ) {
+ when {
+ expression {
+ STATUS == 'Passed'
+ }
+ }
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ }
+ }
+ }
+ }
+}
--
GitLab
From ee7ddad785e1780b4a9db6508be48e1b924b25a5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 19:49:36 +0000
Subject: [PATCH 160/189] updated the name of the job for lanuching in dev
project
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index ff66db85f..03118bd9b 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -44,7 +44,7 @@ pipeline {
// }
// }
// }
- build(job: "EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ build(job: "EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
machine = hera
Machine = machine[0].toUpperCase() + machine.substring(1)
}
--
GitLab
From 661b57e0c3c4e09fcf97b7c1ecc842d2c68afd28 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 19:53:39 +0000
Subject: [PATCH 161/189] added folder name to job name for dev build
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 03118bd9b..648b2119c 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -44,7 +44,7 @@ pipeline {
// }
// }
// }
- build(job: "EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
machine = hera
Machine = machine[0].toUpperCase() + machine.substring(1)
}
--
GitLab
From 66af8090cd7ee497251d49d9b46456e8f820322b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:09:19 +0000
Subject: [PATCH 162/189] stopped recurstion
---
ci/Jenkinsfile_multilabel | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 648b2119c..b364c297f 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -9,7 +9,9 @@ def STATUS = 'Passed'
properties([
parameters([[
- $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'machine', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ string(name: 'machine', defaultValue: 'none', description: 'Node to run on'),
+ buildDiscarder(logRotator(numToKeepStr: '3'))
])
])
@@ -27,6 +29,9 @@ pipeline {
// Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
stage('Get Machine') {
+ when {
+ expression { return machine == 'none' }
+ }
agent { label 'built-in' }
steps {
script {
--
GitLab
From d5c80278d7da6a31da5082198cf553e185b1281a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:12:53 +0000
Subject: [PATCH 163/189] NodeParameterDefinition needed a Name
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index b364c297f..bbb9dbd3d 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -9,7 +9,7 @@ def STATUS = 'Passed'
properties([
parameters([[
- $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
string(name: 'machine', defaultValue: 'none', description: 'Node to run on'),
buildDiscarder(logRotator(numToKeepStr: '3'))
])
--
GitLab
From 0af4e256ee16ebaf356d195143415205a916dff6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:16:15 +0000
Subject: [PATCH 164/189] repete
---
ci/Jenkinsfile_multilabel | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index bbb9dbd3d..9e572af78 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -2,7 +2,7 @@ def Machine = 'none'
def machine = 'none'
def HOME = 'none'
def caseList = ''
-// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
+// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
def STATUS = 'Passed'
@@ -11,7 +11,6 @@ properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
string(name: 'machine', defaultValue: 'none', description: 'Node to run on'),
- buildDiscarder(logRotator(numToKeepStr: '3'))
])
])
--
GitLab
From bc202b892ce7b3d2e32853b7b85a0e66d1f66d2a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:18:37 +0000
Subject: [PATCH 165/189] forgot quotes on hera string
---
ci/Jenkinsfile_multilabel | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 9e572af78..15c143741 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -16,6 +16,8 @@ properties([
pipeline {
+ properties([ buildDiscarder(logRotator(numToKeepStr: '3')) ])
+
agent { label 'built-in' }
options {
@@ -49,7 +51,7 @@ pipeline {
// }
// }
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
- machine = hera
+ machine = 'hera'
Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
--
GitLab
From cc7ea8f8ae2b1d4dc52eca740db05cd00f835122 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:25:23 +0000
Subject: [PATCH 166/189] added a default vaule to machine to orion for testing
---
ci/Jenkinsfile_multilabel | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 15c143741..23578db3c 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -10,7 +10,7 @@ def STATUS = 'Passed'
properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
- string(name: 'machine', defaultValue: 'none', description: 'Node to run on'),
+ string(name: 'machine', defaultValue: 'orion', description: 'Node to run on'),
])
])
@@ -52,7 +52,6 @@ pipeline {
// }
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
machine = 'hera'
- Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
}
@@ -61,6 +60,7 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
+ Machine = machine[0].toUpperCase() + machine.substring(1)
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
--
GitLab
From ed4abf07051ccfd9526a1ed9d70c96f5fae28d8a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:26:22 +0000
Subject: [PATCH 167/189] removed this test case for the Jenkins file and
moving to a new PR
---
ci/Jenkinsfile_multilabel | 263 --------------------------------------
1 file changed, 263 deletions(-)
delete mode 100644 ci/Jenkinsfile_multilabel
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
deleted file mode 100644
index 23578db3c..000000000
--- a/ci/Jenkinsfile_multilabel
+++ /dev/null
@@ -1,263 +0,0 @@
-def Machine = 'none'
-def machine = 'none'
-def HOME = 'none'
-def caseList = ''
-// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
-def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
-def STATUS = 'Passed'
-
-properties([
- parameters([[
- $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
- string(name: 'machine', defaultValue: 'orion', description: 'Node to run on'),
- ])
-])
-
-pipeline {
-
- properties([ buildDiscarder(logRotator(numToKeepStr: '3')) ])
-
- agent { label 'built-in' }
-
- options {
- skipDefaultCheckout()
- //parallelsAlwaysFailFast()
- }
-
- stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
- // which is used to designate the Nodes in the Jenkins Controler by the agent label
- // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
-
- stage('Get Machine') {
- when {
- expression { return machine == 'none' }
- }
- agent { label 'built-in' }
- steps {
- script {
- // generateBaselineNode = []
- // . for (label in pullRequest.labels) {
- // listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
- // node -> node.getLabelString().contains(label) ? node.name : null
-
- // if ((label.matches(node.getLabelString()+"-(.*)"))) {
- // matchedNode += node.getLabelString()
- // }
-
- // if ((label.matches(node.getLabelString()+"(.*)-emc"))) {
- // generateBaselineNode += node.getLabelString()
- // }
- // }
- // }
- build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
- machine = 'hera'
- }
- }
- }
-
- stage('Get Common Workspace') {
- agent { label "${machine}-emc" }
- steps {
- script {
- Machine = machine[0].toUpperCase() + machine.substring(1)
- ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
- HOME = "${WORKSPACE}"
- sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
- }
- echo "Building and running on ${Machine} in directory ${HOME}"
- }
- }
- }
-
- stage('Build System') {
- matrix {
- agent { label "${machine}-emc" }
- //options {
- // throttle(['global_matrix_build'])
- //}
- axes {
- axis {
- name 'system'
- values 'gfs', 'gefs'
- }
- }
- stages {
- stage('build system') {
- steps {
- script {
- def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
- sh(script: "mkdir -p ${HOMEgfs}")
- ws(HOMEgfs) {
- if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
- sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
- checkout scm
- dir('sorc') {
- sh(script: './link_workflow.sh')
- }
- } else {
- checkout scm
- def gist_url = ""
- def error_logs = ""
- def error_logs_message = ""
- def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
- def build_args_list = builds_file['builds']
- def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
- dir("${HOMEgfs}/sorc") {
- try {
- sh(script: "${build_args}")
- } catch (Exception error_build) {
- echo "Failed to build system: ${error_build.getMessage()}"
- if ( fileExists("logs/error.logs") ) {
- def fileContent = readFile 'logs/error.logs'
- def lines = fileContent.readLines()
- for (line in lines) {
- echo "archiving: ${line}"
- if (fileExists("${line}") && readFile("${line}").length() > 0 ){
- try {
- archiveArtifacts artifacts: "${line}", fingerprint: true
- error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
- error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
- }
- catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
- }
- }
- try {
- sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
- gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
- } catch (Exception error_comment) {
- echo "Failed to comment on PR: ${error_comment.getMessage()}"
- }
- error("Failed to build system on ${Machine}")
- }
- }
- sh(script: './link_workflow.sh')
- sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
- }
- }
- if (env.CHANGE_ID && system == 'gfs') {
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
- } catch (Exception e) {
- echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
- }
- }
- if (system == 'gfs') {
- caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
- }
- }
- }
- }
- }
- }
- }
- }
-
- stage('Run Tests') {
- failFast false
- matrix {
- agent { label "${machine}-emc" }
- axes {
- axis {
- name 'Case'
- // TODO add dynamic list of cases from env vars (needs addtional plugins)
- values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
- }
- }
- stages {
-
- stage('Create Experiments') {
- when {
- expression { return caseList.contains(Case) }
- }
- steps {
- script {
- sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
- def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
- system = yaml_case.experiment.system
- def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
- env.RUNTESTS = "${HOME}/RUNTESTS"
- sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml")
- }
- }
- }
-
- stage('Run Experiments') {
- when {
- expression { return caseList.contains(Case) }
- }
- failFast false
- steps {
- script {
- HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
- def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
- try {
- sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
- } catch (Exception error_experment) {
- sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
- ws(HOME) {
- def error_logs = ""
- def error_logs_message = ""
- def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
- if (fileExists(error_file)) {
- def fileContent = readFile error_file
- def lines = fileContent.readLines()
- for (line in lines) {
- echo "archiving: ${line}"
- if (fileExists("${HOME}/${line}") && readFile("${HOME}/${line}").length() > 0) {
- try {
- archiveArtifacts artifacts: "${line}", fingerprint: true
- error_logs = error_logs + "${HOME}/${line} "
- error_logs_message = error_logs_message + "${HOME}/${line}\n"
- } catch (Exception error_arch) {
- echo "Failed to archive error log ${line}: ${error_arch.getMessage()}"
- }
- }
- }
- try {
- gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
- sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
- } catch (Exception error_comment) {
- echo "Failed to comment on PR: ${error_comment.getMessage()}"
- }
- } else {
- echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
- }
- STATUS = 'Failed'
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
- } catch (Exception e) {
- echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
- }
- error("Failed to run experiments ${Case} on ${Machine}")
- }
- }
- }
- }
- }
- }
- }
- }
- stage( 'FINALIZE' ) {
- when {
- expression {
- STATUS == 'Passed'
- }
- }
- agent { label "${machine}-emc" }
- steps {
- script {
- try {
- sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
- sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
- } catch (Exception e) {
- echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
- }
- }
- }
- }
- }
-}
--
GitLab
From 551dd4347a047ef5895a58cf61763621ea2b2130 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:33:14 +0000
Subject: [PATCH 168/189] forgot to commint test Jeninks file
---
ci/Jenkinsfile_multilabel | 263 ++++++++++++++++++++++++++++++++++++++
1 file changed, 263 insertions(+)
create mode 100644 ci/Jenkinsfile_multilabel
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
new file mode 100644
index 000000000..23578db3c
--- /dev/null
+++ b/ci/Jenkinsfile_multilabel
@@ -0,0 +1,263 @@
+def Machine = 'none'
+def machine = 'none'
+def HOME = 'none'
+def caseList = ''
+// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
+def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
+def STATUS = 'Passed'
+
+properties([
+ parameters([[
+ $class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
+ string(name: 'machine', defaultValue: 'orion', description: 'Node to run on'),
+ ])
+])
+
+pipeline {
+
+ properties([ buildDiscarder(logRotator(numToKeepStr: '3')) ])
+
+ agent { label 'built-in' }
+
+ options {
+ skipDefaultCheckout()
+ //parallelsAlwaysFailFast()
+ }
+
+ stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
+ // which is used to designate the Nodes in the Jenkins Controler by the agent label
+ // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+
+ stage('Get Machine') {
+ when {
+ expression { return machine == 'none' }
+ }
+ agent { label 'built-in' }
+ steps {
+ script {
+ // generateBaselineNode = []
+ // . for (label in pullRequest.labels) {
+ // listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
+ // node -> node.getLabelString().contains(label) ? node.name : null
+
+ // if ((label.matches(node.getLabelString()+"-(.*)"))) {
+ // matchedNode += node.getLabelString()
+ // }
+
+ // if ((label.matches(node.getLabelString()+"(.*)-emc"))) {
+ // generateBaselineNode += node.getLabelString()
+ // }
+ // }
+ // }
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ machine = 'hera'
+ }
+ }
+ }
+
+ stage('Get Common Workspace') {
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ Machine = machine[0].toUpperCase() + machine.substring(1)
+ ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
+ HOME = "${WORKSPACE}"
+ sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
+ }
+ echo "Building and running on ${Machine} in directory ${HOME}"
+ }
+ }
+ }
+
+ stage('Build System') {
+ matrix {
+ agent { label "${machine}-emc" }
+ //options {
+ // throttle(['global_matrix_build'])
+ //}
+ axes {
+ axis {
+ name 'system'
+ values 'gfs', 'gefs'
+ }
+ }
+ stages {
+ stage('build system') {
+ steps {
+ script {
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
+ sh(script: "mkdir -p ${HOMEgfs}")
+ ws(HOMEgfs) {
+ if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
+ sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
+ checkout scm
+ dir('sorc') {
+ sh(script: './link_workflow.sh')
+ }
+ } else {
+ checkout scm
+ def gist_url = ""
+ def error_logs = ""
+ def error_logs_message = ""
+ def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
+ def build_args_list = builds_file['builds']
+ def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
+ dir("${HOMEgfs}/sorc") {
+ try {
+ sh(script: "${build_args}")
+ } catch (Exception error_build) {
+ echo "Failed to build system: ${error_build.getMessage()}"
+ if ( fileExists("logs/error.logs") ) {
+ def fileContent = readFile 'logs/error.logs'
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${line}") && readFile("${line}").length() > 0 ){
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
+ error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
+ }
+ catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
+ }
+ }
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+ gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ error("Failed to build system on ${Machine}")
+ }
+ }
+ sh(script: './link_workflow.sh')
+ sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+ }
+ }
+ if (env.CHANGE_ID && system == 'gfs') {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
+ } catch (Exception e) {
+ echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+ }
+ }
+ if (system == 'gfs') {
+ caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ stage('Run Tests') {
+ failFast false
+ matrix {
+ agent { label "${machine}-emc" }
+ axes {
+ axis {
+ name 'Case'
+ // TODO add dynamic list of cases from env vars (needs addtional plugins)
+ values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
+ }
+ }
+ stages {
+
+ stage('Create Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ steps {
+ script {
+ sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
+ def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
+ system = yaml_case.experiment.system
+ def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
+ env.RUNTESTS = "${HOME}/RUNTESTS"
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml")
+ }
+ }
+ }
+
+ stage('Run Experiments') {
+ when {
+ expression { return caseList.contains(Case) }
+ }
+ failFast false
+ steps {
+ script {
+ HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
+ def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
+ try {
+ sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}")
+ } catch (Exception error_experment) {
+ sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
+ ws(HOME) {
+ def error_logs = ""
+ def error_logs_message = ""
+ def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs"
+ if (fileExists(error_file)) {
+ def fileContent = readFile error_file
+ def lines = fileContent.readLines()
+ for (line in lines) {
+ echo "archiving: ${line}"
+ if (fileExists("${HOME}/${line}") && readFile("${HOME}/${line}").length() > 0) {
+ try {
+ archiveArtifacts artifacts: "${line}", fingerprint: true
+ error_logs = error_logs + "${HOME}/${line} "
+ error_logs_message = error_logs_message + "${HOME}/${line}\n"
+ } catch (Exception error_arch) {
+ echo "Failed to archive error log ${line}: ${error_arch.getMessage()}"
+ }
+ }
+ }
+ try {
+ gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+ sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+ } catch (Exception error_comment) {
+ echo "Failed to comment on PR: ${error_comment.getMessage()}"
+ }
+ } else {
+ echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs"
+ }
+ STATUS = 'Failed'
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ error("Failed to run experiments ${Case} on ${Machine}")
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ stage( 'FINALIZE' ) {
+ when {
+ expression {
+ STATUS == 'Passed'
+ }
+ }
+ agent { label "${machine}-emc" }
+ steps {
+ script {
+ try {
+ sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+ sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true)
+ } catch (Exception e) {
+ echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
+ }
+ }
+ }
+ }
+ }
+}
--
GitLab
From 59ecfe0a12b00d0077e9623fb2e87aef98609a4c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:38:57 +0000
Subject: [PATCH 169/189] added buildDiscarder to options
---
ci/Jenkinsfile_multilabel | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 23578db3c..626cbf07c 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -16,12 +16,11 @@ properties([
pipeline {
- properties([ buildDiscarder(logRotator(numToKeepStr: '3')) ])
-
agent { label 'built-in' }
options {
skipDefaultCheckout()
+ buildDiscarder(logRotator(numToKepStr: '3'))
//parallelsAlwaysFailFast()
}
--
GitLab
From bbc0e0b3f7fcf325df0fb74383f0c48fd46d42a7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:41:54 +0000
Subject: [PATCH 170/189] added buildDiscarder to options
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 626cbf07c..cc2787ba1 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -20,7 +20,7 @@ pipeline {
options {
skipDefaultCheckout()
- buildDiscarder(logRotator(numToKepStr: '3'))
+ buildDiscarder(logRotator(numToKepStr: '1'))
//parallelsAlwaysFailFast()
}
--
GitLab
From 638d1e0f2cfd4ed0a34601aa5ca06cfe2751daae Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:42:54 +0000
Subject: [PATCH 171/189] added a no op
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index cc2787ba1..a4d8705e9 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -10,7 +10,7 @@ def STATUS = 'Passed'
properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
- string(name: 'machine', defaultValue: 'orion', description: 'Node to run on'),
+ string(name: 'machine', defaultValue: 'orion', description: 'Node to run on today'),
])
])
--
GitLab
From f7c8f624bda8bea4a09cf211b4206abdec56e8ba Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:44:24 +0000
Subject: [PATCH 172/189] its keep not kep
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index a4d8705e9..274376b7a 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -20,7 +20,7 @@ pipeline {
options {
skipDefaultCheckout()
- buildDiscarder(logRotator(numToKepStr: '1'))
+ buildDiscarder(logRotator(numToKeepStr: '1'))
//parallelsAlwaysFailFast()
}
--
GitLab
From 100108c0d36972a874f58e69c3e3d417774d8228 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 20:59:57 +0000
Subject: [PATCH 173/189] try to wait to see it works att all
---
ci/Jenkinsfile_multilabel | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 274376b7a..ebf511f32 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -10,7 +10,7 @@ def STATUS = 'Passed'
properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
- string(name: 'machine', defaultValue: 'orion', description: 'Node to run on today'),
+ string(name: 'machine', defaultValue: 'nobody', description: 'Node to run on today'),
])
])
@@ -49,7 +49,7 @@ pipeline {
// }
// }
// }
- build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: true)
machine = 'hera'
}
}
--
GitLab
From 78b5b11dc90b477e18272f00a192e70cb3e1ce94 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 18 Apr 2024 22:02:11 +0000
Subject: [PATCH 174/189] checking if initial properties is setting macine
---
ci/Jenkinsfile_multilabel | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index ebf511f32..470f17dee 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -10,7 +10,7 @@ def STATUS = 'Passed'
properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
- string(name: 'machine', defaultValue: 'nobody', description: 'Node to run on today'),
+ string(name: 'machine', defaultValue: 'orion', description: 'Node to run on today'),
])
])
@@ -20,7 +20,7 @@ pipeline {
options {
skipDefaultCheckout()
- buildDiscarder(logRotator(numToKeepStr: '1'))
+ buildDiscarder(logRotator(numToKeepStr: '3'))
//parallelsAlwaysFailFast()
}
--
GitLab
From 539408342cb42dde329c53a0ca379dac0c4d17c5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:01:42 +0000
Subject: [PATCH 175/189] change wait to faluse
---
ci/Jenkinsfile_multilabel | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 470f17dee..9b9fc8b54 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -29,9 +29,6 @@ pipeline {
// Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
stage('Get Machine') {
- when {
- expression { return machine == 'none' }
- }
agent { label 'built-in' }
steps {
script {
@@ -49,8 +46,10 @@ pipeline {
// }
// }
// }
- build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: true)
- machine = 'hera'
+ if (machine == 'none') {
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ machine = 'hera'
+ }
}
}
}
--
GitLab
From 202da38adbc037a8012406ec17fa39b05ff61c5e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:22:42 +0000
Subject: [PATCH 176/189] passed also Node name to build job
---
ci/Jenkinsfile_multilabel | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 9b9fc8b54..66e6578cd 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -47,7 +47,11 @@ pipeline {
// }
// }
if (machine == 'none') {
- build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}", parameters: [string(name: 'machine', value: 'orion')], wait: false)
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
+ parameters: [string(name: 'machine', value: 'orion')],
+ [string(name: 'Nod', value: 'orion-emc')],
+ wait: false
+ )
machine = 'hera'
}
}
--
GitLab
From 3484d726822371eff205127d94ef24e9ae5927ae Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:26:24 +0000
Subject: [PATCH 177/189] got brackets wrog when lauching
---
ci/Jenkinsfile_multilabel | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 66e6578cd..238dee155 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -48,8 +48,7 @@ pipeline {
// }
if (machine == 'none') {
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [string(name: 'machine', value: 'orion')],
- [string(name: 'Nod', value: 'orion-emc')],
+ parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Nod', value: 'orion-emc') ],
wait: false
)
machine = 'hera'
--
GitLab
From fe1557f5764bdef6b7af5e2ba1afbdbd839a9a11 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:32:37 +0000
Subject: [PATCH 178/189] misspelled Node and add error print of machine for
checkout
---
ci/Jenkinsfile_multilabel | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 238dee155..5a1ab4e4e 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -46,9 +46,10 @@ pipeline {
// }
// }
// }
+ echo "machine: ${machine}"
if (machine == 'none') {
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Nod', value: 'orion-emc') ],
+ parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Node', value: 'orion-emc') ],
wait: false
)
machine = 'hera'
--
GitLab
From b727de0397809e7fa36bc6fe015a19933f2ef3e9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:40:56 +0000
Subject: [PATCH 179/189] did not have the correct string for the Node
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 5a1ab4e4e..2f7444a8d 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -49,7 +49,7 @@ pipeline {
echo "machine: ${machine}"
if (machine == 'none') {
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Node', value: 'orion-emc') ],
+ parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Node', value: 'Orion-EMC') ],
wait: false
)
machine = 'hera'
--
GitLab
From 58a764b1e3a8c576c2fd568ca8eb87c79d593ac8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 19:47:42 +0000
Subject: [PATCH 180/189] Updated URL to forked repos for development
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 2f7444a8d..143fec504 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -4,7 +4,7 @@ def HOME = 'none'
def caseList = ''
// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
-def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
+def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
def STATUS = 'Passed'
properties([
--
GitLab
From 93b028bacb394d6dc4d9f140356544d7b5737f41 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 19 Apr 2024 20:05:12 +0000
Subject: [PATCH 181/189] tried setting before launching, can not belive thiat
was it
---
ci/Jenkinsfile_multilabel | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 143fec504..39b68ca1b 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -48,8 +48,9 @@ pipeline {
// }
echo "machine: ${machine}"
if (machine == 'none') {
+ machine = 'orion'
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: 'orion'), string(name: 'Node', value: 'Orion-EMC') ],
+ parameters: [ string(name: 'machine', value: machine), Node(name: 'Node', value: 'Orion-EMC') ],
wait: false
)
machine = 'hera'
--
GitLab
From 3b4ca88a27d0663ce046fcf495f6ddad049f5b11 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 00:46:31 +0000
Subject: [PATCH 182/189] remove global scope of the machine variable that is
being passed into spawned job
---
ci/Jenkinsfile_multilabel | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 39b68ca1b..6749f1f1f 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -1,5 +1,5 @@
def Machine = 'none'
-def machine = 'none'
+// def machine = 'none'
def HOME = 'none'
def caseList = ''
// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
@@ -54,6 +54,7 @@ pipeline {
wait: false
)
machine = 'hera'
+ Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
}
@@ -63,7 +64,6 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
- Machine = machine[0].toUpperCase() + machine.substring(1)
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
--
GitLab
From 4f89b204c9c447b5cbdd95afe6999f55c31b7b9c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 00:56:41 +0000
Subject: [PATCH 183/189] using check to see if machine is defined
---
ci/Jenkinsfile_multilabel | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 6749f1f1f..aece06797 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -46,16 +46,15 @@ pipeline {
// }
// }
// }
- echo "machine: ${machine}"
- if (machine == 'none') {
+ if (!binding.hasVariable('machine')) {
machine = 'orion'
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
parameters: [ string(name: 'machine', value: machine), Node(name: 'Node', value: 'Orion-EMC') ],
wait: false
)
- machine = 'hera'
- Machine = machine[0].toUpperCase() + machine.substring(1)
}
+ machine = 'hera'
+ Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
}
--
GitLab
From c6c2974f47fdfc8172e49b50084ca8e6dc801fc5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:00:05 +0000
Subject: [PATCH 184/189] still had node laying around
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index aece06797..52f28157a 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -49,7 +49,7 @@ pipeline {
if (!binding.hasVariable('machine')) {
machine = 'orion'
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: machine), Node(name: 'Node', value: 'Orion-EMC') ],
+ parameters: [ string(name: 'machine', value: machine), string(name: 'Node', value: 'Orion-EMC') ],
wait: false
)
}
--
GitLab
From 0315d47d4a8771a40ebbcb77fbcd49fff3a53a65 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:16:27 +0000
Subject: [PATCH 185/189] using when expressin for machine not defined
---
ci/Jenkinsfile_multilabel | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 52f28157a..29c11a02a 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -29,6 +29,9 @@ pipeline {
// Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
stage('Get Machine') {
+ when {
+ expression { !binding.hasVariable('machine') }
+ }
agent { label 'built-in' }
steps {
script {
@@ -46,13 +49,12 @@ pipeline {
// }
// }
// }
- if (!binding.hasVariable('machine')) {
- machine = 'orion'
- build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: machine), string(name: 'Node', value: 'Orion-EMC') ],
- wait: false
+ machine = 'orion'
+ build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
+ parameters: [ string(name: 'machine', value: machine),
+ string(name: 'Node', value: 'Orion-EMC') ],
+ wait: false
)
- }
machine = 'hera'
Machine = machine[0].toUpperCase() + machine.substring(1)
}
--
GitLab
From 914a86af6948614c8734d3b9046b8270c7141984 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:27:42 +0000
Subject: [PATCH 186/189] start orion and spawn hera
---
ci/Jenkinsfile_multilabel | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 29c11a02a..d4a820480 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -49,14 +49,12 @@ pipeline {
// }
// }
// }
- machine = 'orion'
build(job: "global-workflow/EMC-Pipelines/PR-${env.CHANGE_ID}",
- parameters: [ string(name: 'machine', value: machine),
- string(name: 'Node', value: 'Orion-EMC') ],
+ parameters: [ string(name: 'machine', value: 'hera'),
+ string(name: 'Node', value: 'Hera-EMC') ],
wait: false
)
- machine = 'hera'
- Machine = machine[0].toUpperCase() + machine.substring(1)
+ machine = 'orion'
}
}
}
@@ -65,6 +63,7 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
+ Machine = machine[0].toUpperCase() + machine.substring(1)
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
--
GitLab
From f10af355ea0fcb23f495e00cb2c7721487b908e0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:39:11 +0000
Subject: [PATCH 187/189] start orion and spawn hera
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index d4a820480..4d88df8d9 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -1,5 +1,5 @@
def Machine = 'none'
-// def machine = 'none'
+// def machine = 'none np'
def HOME = 'none'
def caseList = ''
// no op Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
--
GitLab
From e59f25efa6c561863ffc3651724c2d0bdafe3bf8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:46:30 +0000
Subject: [PATCH 188/189] usng expression with BUILD_CAUSE
---
ci/Jenkinsfile_multilabel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index 4d88df8d9..a37daa1dc 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -30,7 +30,7 @@ pipeline {
stage('Get Machine') {
when {
- expression { !binding.hasVariable('machine') }
+ expression { env.BUILD_CAUSE != 'UPSTREAMTRIGGER' }
}
agent { label 'built-in' }
steps {
--
GitLab
From 5bd20df4783ad78c257e8e79324f1389300c1227 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Sat, 20 Apr 2024 01:54:04 +0000
Subject: [PATCH 189/189] moved properties scope
---
ci/Jenkinsfile_multilabel | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ci/Jenkinsfile_multilabel b/ci/Jenkinsfile_multilabel
index a37daa1dc..230f45574 100644
--- a/ci/Jenkinsfile_multilabel
+++ b/ci/Jenkinsfile_multilabel
@@ -7,6 +7,8 @@ def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa
def repo_url = 'git@github.com:TerrenceMcGuinness-NOAA/global-workflow.git'
def STATUS = 'Passed'
+pipeline {
+
properties([
parameters([[
$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: 'Node', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases'],
@@ -14,8 +16,6 @@ properties([
])
])
-pipeline {
-
agent { label 'built-in' }
options {
--
GitLab