Skip to content

Commit

Permalink
Merge branch 'v2.x/staging' into v2.x/zweLogging
Browse files Browse the repository at this point in the history
  • Loading branch information
MarkAckert authored Nov 4, 2024
2 parents 1d9ed56 + ecc8abe commit d655b17
Show file tree
Hide file tree
Showing 35 changed files with 623 additions and 661 deletions.
10 changes: 5 additions & 5 deletions .github/scripts/cicd_test/make_matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ case $install_test_choice in

"z/OS node v18")
test_file="$ZOS_NODE_V18_TESTFILE"
test_force_system="zzow08"
test_force_system="zzow11"
;;

"z/OS node v20")
Expand Down Expand Up @@ -87,18 +87,18 @@ if [[ ! -z "$test_force_system" ]]; then
else
if [[ -z "$dont_parse_test_server" ]]; then
if [[ "$test_server" == "Any zzow servers" ]]; then
test_server="zzow0"$(echo $(($RANDOM % 3 + 6)))
test_server="zzow"$(printf %02d $(($RANDOM % 3 + 9)))
fi
TEST_FILE_SERVER="$test_file($test_server)"
else
any_occurrence=$(echo $test_file | grep -o "(any)" | wc -l)
interim_test_file_server=$test_file
for i in $(seq $any_occurrence); do
# Generates zzow06, zzow07, zzow08
interim_test_file_server=$(echo $interim_test_file_server | sed "s#(any)#(zzow0$(echo $(($RANDOM % 3 + 6))))#")
# Generates zzow09, zzow10, zzow11
interim_test_file_server=$(echo $interim_test_file_server | sed "s#(any)#(zzow$(printf %02d $(($RANDOM % 3 + 9))))#")
done

TEST_FILE_SERVER=$(echo $interim_test_file_server | sed "s#(all)#(zzow06,zzow07,zzow08)#g")
TEST_FILE_SERVER=$(echo $interim_test_file_server | sed "s#(all)#(zzow09,zzow10,zzow11)#g")
fi
fi

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@ TEST_SERVER=$(echo "$MATRIX_SERVER" | cut -d "-" -f2)

case $TEST_SERVER in

"zzow06")
TEST_SERVER_NICKNAME=marist-6
"zzow09")
TEST_SERVER_NICKNAME=marist-9
;;

"zzow07")
TEST_SERVER_NICKNAME=marist-7
"zzow10")
TEST_SERVER_NICKNAME=marist-10
;;

"zzow08")
TEST_SERVER_NICKNAME=marist-8
"zzow11")
TEST_SERVER_NICKNAME=marist-11
;;

*)
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/build-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ jobs:
echo "run_build=${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.ORIGIN_ISSUE_TRIGGER == 'false') || !contains(fromJson(steps.get-labels.outputs.result), 'Build: None') }}" >> $GITHUB_OUTPUT
- id: check-test
name: 'export conditional used to determine if we should run a test suite'
# run_test explanation: if we set "run_build" from the prior step and this is a PR, run unless "Test: None" label is present. If not initiated from a PR, only build.
run: |
echo "run_test=${{ (steps.check-build.outputs.run_build == 'true' && !contains(fromJson(steps.get-labels.outputs.result), 'Test: None')) }}" >> $GITHUB_OUTPUT
echo "run_test=${{ (steps.check-build.outputs.run_build == 'true' && github.event_name != 'workflow_dispatch' && !contains(fromJson(steps.get-labels.outputs.result), 'Test: None')) }}" >> $GITHUB_OUTPUT
display-dispatch-event-id:
if: github.event.inputs.RANDOM_DISPATCH_EVENT_ID != ''
Expand Down Expand Up @@ -344,7 +345,7 @@ jobs:
with:
lock-repository: ${{ github.repository }}
github-token: ${{ secrets.GITHUB_TOKEN }}
lock-resource-name: zowe-psi-build-zzow07-lock
lock-resource-name: zowe-psi-build-zzow10-lock
lock-avg-retry-interval: 60

- name: '[PSWI 0] PSWI pre-build check for existing smpe'
Expand All @@ -367,6 +368,14 @@ jobs:
ZZOW_SSH_PORT: ${{ secrets.SSH_MARIST_ALLSYS_PORT }}
VERSION: ${{ env.P_VERSION }}

- name: Store PSWI folder
uses: actions/upload-artifact@v4
if: env.INPUTS_BUILD_PSWI == 'true' && failure()
with:
name: pswi-folder
path: |
pswi/logs/**
- name: '[K8S] Build Kubernetes'
timeout-minutes: 10
if: env.INPUTS_BUILD_KUBERNETES == 'true'
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/cicd-test-readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ This guide will describe how you should input into Github Actions workflow input

Currently we support three testing z/OS servers:

- zzow06 (ACF2)
- zzow07 (Top Secret/TSS)
- zzow08 (RACF)
- zzow09 (ACF2)
- zzow10 (Top Secret/TSS)
- zzow11 (RACF)

Testing pipeline is running tests in parallel. The workflow will try to acquire the resource lock if available. If the resource lock is occupied, the workflow will wait until the lock is succesfully acquired.

Expand All @@ -17,7 +17,7 @@ Workflow trigger is at [cicd-test](https://github.com/zowe/zowe-install-packagin
### Choose Test Server

- This input is a choice, and it's mandatory.
- You can choose from one of `zzow06`, `zzow07`, `zzow08`, `zzow06,zzow07,zzow08` (if you want to run the test on all zzow servers), or `Any zzow servers` (pick any zzow servers, potentially help reduce wait time)
- You can choose from one of `zzow09`, `zzow10`, `zzow11`, `zzow09,zzow10,zzow11` (if you want to run the test on all zzow servers), or `Any zzow servers` (pick any zzow servers, potentially help reduce wait time)
- Default is `Any zzow servers`

### Choose Install Test
Expand Down Expand Up @@ -109,7 +109,7 @@ Background: CICD testing relies on a `zowe.pax` or `zowe-smpe.zip` (for SMPE ins
When running CICD integration tests during RC stage, the following string will be parsed into the Github Actions matrix. As a result, a total of 21 independent jobs will be spawned.
```
basic/install.ts(zzow06,zzow07,zzow08);basic/install-ptf.ts(zzow06,zzow07,zzow08);basic/install-ext.ts(zzow07);extended/keyring.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v18.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v20.ts(zzow06,zzow07,zzow08);extended/certificates/nonstrict-verify-external-certificate.ts(zzow06)
basic/install.ts(zzow09,zzow10,zzow11);basic/install-ptf.ts(zzow09,zzow10,zzow11);basic/install-ext.ts(zzow10);extended/keyring.ts(zzow09,zzow10,zzow11);extended/node-versions/node-v18.ts(zzow09,zzow10,zzow11);extended/node-versions/node-v20.ts(zzow09,zzow10,zzow11);extended/certificates/nonstrict-verify-external-certificate.ts(zzow09)
```
Total elapsed time when running in parallel is approximately 3.5 hours on paper idealy if all parallel jobs are executing at the same time. In reality, from numerous tests performed, total elapsed time is around 4 hours.
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/cicd-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ on:
default: 'Any zzow servers'
options:
- Any zzow servers
- zzow06
- zzow07
- zzow08
- zzow06,zzow07,zzow08
- zzow09
- zzow10
- zzow11
- zzow09,zzow10,zzow11
install-test:
description: 'Choose Install Test'
type: choice
Expand Down Expand Up @@ -90,7 +90,7 @@ env:
CONFIG_MANAGER_TESTFILE: extended/config-manager/enable-config-manager.ts
GENERAL_API_DOCUMENTATION_TESTFILE: basic/install-api-gen.ts
ZOWE_NIGHTLY_TESTS_FULL: basic/install.ts(all);basic/install-ptf.ts(all)
ZOWE_RELEASE_TESTS_FULL: basic/install.ts(all);basic/install-ptf.ts(all);basic/install-ext.ts(any);extended/keyring.ts(all);extended/node-versions/node-v18.ts(zzow08);extended/node-versions/node-v20.ts(any);extended/certificates/nonstrict-verify-external-certificate.ts(any);extended/caching-storages/infinispan-storage.ts(any);extended/config-manager/enable-config-manager.ts(any)
ZOWE_RELEASE_TESTS_FULL: basic/install.ts(all);basic/install-ptf.ts(all);basic/install-ext.ts(any);extended/keyring.ts(all);extended/node-versions/node-v18.ts(zzow11);extended/node-versions/node-v20.ts(any);extended/certificates/nonstrict-verify-external-certificate.ts(any);extended/caching-storages/infinispan-storage.ts(any);extended/config-manager/enable-config-manager.ts(any)

jobs:
display-dispatch-event-id:
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
6 changes: 3 additions & 3 deletions playbooks/hosts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[marist]
marist-6
marist-7
marist-8
marist-9
marist-10
marist-11

[tvt]
tvt4188
Expand Down
4 changes: 4 additions & 0 deletions pswi/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
logs/
EXP*
report.txt
unzipped/
66 changes: 30 additions & 36 deletions pswi/01_smpe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ echo "SMPE workflow name :" $SMPE_WF_NAME
CREATE_SMPE_WF_URL="${BASE_URL}/zosmf/workflow/rest/1.0/workflows"
SMPE_WF_LIST_URL="${BASE_URL}/zosmf/workflow/rest/1.0/workflows?owner=${ZOSMF_USER}&workflowName=${SMPE_WF_NAME}"

# JSONs
# JSONs

ADD_WORKFLOW_JSON='{"workflowName":"'$SMPE_WF_NAME'",
"workflowDefinitionFile":"'${DIR}'/SMPE20",
Expand All @@ -43,7 +43,7 @@ echo "Uploading workflow SMPE into ${DIR} directory thru SSH"

cd workflows

sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} <<EOF
cd ${DIR}
put SMPE20
EOF
Expand All @@ -52,52 +52,46 @@ cd ..
# Get workflowKey for SMPE workflow owned by user
echo "Get workflowKey for SMPE workflow if it exists."

RESP=`curl -s $SMPE_WF_LIST_URL -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
WFKEY=`echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\"`
if [ -n "$WFKEY" ]
then
SMPE_WORKFLOW_URL="${CREATE_SMPE_WF_URL}/${WFKEY}"
RESP=$(curl -s $SMPE_WF_LIST_URL -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
WFKEY=$(echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\")
if [ -n "$WFKEY" ]; then
SMPE_WORKFLOW_URL="${CREATE_SMPE_WF_URL}/${WFKEY}"

echo "Deleting an SMPE workflow."
RESP=`curl -s $SMPE_WORKFLOW_URL -k -X "DELETE" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
sh scripts/check_response.sh "${RESP}" $?
echo "Deleting an SMPE workflow."
RESP=$(curl -s $SMPE_WORKFLOW_URL -k -X "DELETE" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
fi

# Create workflow with REST API
echo 'Invoking REST API to create SMPE workflow.'

RESP=`curl -s $CREATE_SMPE_WF_URL -k -X "POST" -d "$ADD_WORKFLOW_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
RESP=$(curl -s $CREATE_SMPE_WF_URL -k -X "POST" -d "$ADD_WORKFLOW_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
if [ $? -gt 0 ];then exit -1;fi
WFKEY=`echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\"`
if [ $? -gt 0 ]; then exit -1; fi
WFKEY=$(echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\")
WORKFLOW_URL="${CREATE_SMPE_WF_URL}/${WFKEY}"

# Run workflow
echo "Invoking REST API to start a SMPE workflow."

RESP=`curl -s ${WORKFLOW_URL}/operations/start -k -X "PUT" -d "{}" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
RESP=$(curl -s ${WORKFLOW_URL}/operations/start -k -X "PUT" -d "{}" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
if [ $? -gt 0 ];then exit -1;fi
if [ $? -gt 0 ]; then exit -1; fi
STATUS=""
until [ "$STATUS" = "FINISHED" ]
do
sleep 20


# Get the result of the workflow
RESP=`curl -s ${WORKFLOW_URL} -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
if [ $? -gt 0 ];then exit -1;fi
STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`

if [ "$STATUS_NAME" = "in-progress" ]
then
echo "Workflow ended with an error."
echo $RESP
exit -1
elif [ "$STATUS_NAME" = "complete" ]
then
echo "Workflow finished successfully."
STATUS="FINISHED"
fi
until [ "$STATUS" = "FINISHED" ]; do
sleep 20

# Get the result of the workflow
RESP=$(curl -s ${WORKFLOW_URL} -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
if [ $? -gt 0 ]; then exit -1; fi
STATUS_NAME=$(echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\")

if [ "$STATUS_NAME" = "in-progress" ]; then
echo "Workflow with SMP/E ended with an error." >>$LOG_DIR/report.txt
echo $RESP >>$LOG_DIR/report.txt
exit -1
elif [ "$STATUS_NAME" = "complete" ]; then
echo "Workflow finished successfully."
STATUS="FINISHED"
fi
done

66 changes: 31 additions & 35 deletions pswi/02_ptf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ echo "PTF workflow name :" $PTF_WF_NAME
CREATE_PTF_WF_URL="${BASE_URL}/zosmf/workflow/rest/1.0/workflows"
PTF_WF_LIST_URL="${BASE_URL}/zosmf/workflow/rest/1.0/workflows?owner=${ZOSMF_USER}&workflowName=${PTF_WF_NAME}"

# JSONs
# JSONs
ADD_WORKFLOW_JSON='{"workflowName":"'$PTF_WF_NAME'",
"workflowDefinitionFile":"'${DIR}'/WFPTF",
"system":"'$ZOSMF_SYSTEM'",
Expand All @@ -38,7 +38,7 @@ ADD_WORKFLOW_JSON='{"workflowName":"'$PTF_WF_NAME'",

cd workflows

sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} <<EOF
cd ${DIR}
put WFPTF
EOF
Expand All @@ -47,52 +47,48 @@ cd ..
# Get workflowKey for PTF workflow owned by user
echo "Get workflowKey for PTF workflow if it exists."

RESP=`curl -s $PTF_WF_LIST_URL -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
WFKEY=`echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\"`
RESP=$(curl -s $PTF_WF_LIST_URL -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
WFKEY=$(echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\")

if [ -n "$WFKEY" ]
then
PTF_WORKFLOW_URL="${CREATE_PTF_WF_URL}/${WFKEY}"
if [ -n "$WFKEY" ]; then
PTF_WORKFLOW_URL="${CREATE_PTF_WF_URL}/${WFKEY}"

echo "Deleting a PTF workflow."
RESP=`curl -s $PTF_WORKFLOW_URL -k -X "DELETE" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
sh scripts/check_response.sh "${RESP}" $?
echo "Deleting a PTF workflow."
RESP=$(curl -s $PTF_WORKFLOW_URL -k -X "DELETE" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
fi

# Create workflow with REST API
echo 'Invoking REST API to create ptf workflow.'

RESP=`curl -s $CREATE_PTF_WF_URL -k -X "POST" -d "$ADD_WORKFLOW_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
RESP=$(curl -s $CREATE_PTF_WF_URL -k -X "POST" -d "$ADD_WORKFLOW_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
if [ $? -gt 0 ];then exit -1;fi
WFKEY=`echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\"`
if [ $? -gt 0 ]; then exit -1; fi
WFKEY=$(echo $RESP | grep -o '"workflowKey":".*"' | cut -f4 -d\")
WORKFLOW_URL="${CREATE_PTF_WF_URL}/${WFKEY}"

# Run workflow
echo "Invoking REST API to start a PTF apply workflow."

RESP=`curl -s ${WORKFLOW_URL}/operations/start -k -X "PUT" -d "{}" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
RESP=$(curl -s ${WORKFLOW_URL}/operations/start -k -X "PUT" -d "{}" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
sh scripts/check_response.sh "${RESP}" $?
if [ $? -gt 0 ];then exit -1;fi
if [ $? -gt 0 ]; then exit -1; fi
STATUS=""
until [ "$STATUS" = "FINISHED" ]
do
sleep 20

# Get the result of the workflow
RESP=`curl -s ${WORKFLOW_URL} -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
if [ $? -gt 0 ];then exit -1;fi

STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`

if [ "$STATUS_NAME" = "in-progress" ]
then
echo "Workflow ended with an error."
echo $RESP
exit -1
elif [ "$STATUS_NAME" = "complete" ]
then
echo "Workflow finished successfully."
STATUS="FINISHED"
fi
until [ "$STATUS" = "FINISHED" ]; do
sleep 20

# Get the result of the workflow
RESP=$(curl -s ${WORKFLOW_URL} -k -X "GET" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS)
if [ $? -gt 0 ]; then exit -1; fi

STATUS_NAME=$(echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\")

if [ "$STATUS_NAME" = "in-progress" ]; then
echo "Workflow with PTFs ended with an error." >>$LOG_DIR/report.txt
echo $RESP >>$LOG_DIR/report.txt
exit -1
elif [ "$STATUS_NAME" = "complete" ]; then
echo "Workflow finished successfully."
STATUS="FINISHED"
fi
done
Loading

0 comments on commit d655b17

Please sign in to comment.