User/muzikovam/zowe yaml diff

.github/workflows/build-packaging.yml

@@ -353,8 +353,8 @@ jobs:
           jfrog rt dl ${{github.event.inputs.PSWI_SMPE_AZWE_ARTIFACTORY_PATH}}/AZWE002*.zip --flat=true .pax/AZWE002.zip
           jfrog rt dl ${{github.event.inputs.PSWI_SMPE_ARTIFACTORY_PATH}}/zowe-smpe-*.zip --flat=true .pax/zowe-smpe.zip
 
-
       - name: '[SMPE Pax 4] Build PSWI'
+        id: pswi
         if: env.INPUTS_BUILD_PSWI == 'true'
         timeout-minutes: 60
         run: |
@@ -367,6 +367,14 @@ jobs:
           ZZOW_SSH_PORT: ${{ secrets.SSH_MARIST_ALLSYS_PORT }}
           VERSION: ${{ env.P_VERSION }}
 
+      - name: Store PSWI folder
+        uses: actions/upload-artifact@v4
+        if: env.INPUTS_BUILD_PSWI == 'true' && failure()
+        with:
+          name: pswi-folder
+          path: |
+            pswi/**
+
       - name: '[K8S] Build Kubernetes'
         timeout-minutes: 10
         if: env.INPUTS_BUILD_KUBERNETES == 'true'

.github/workflows/pswi-zowe-config-test.yml

@@ -0,0 +1,52 @@
+name: PSWI zowe.yaml sync check
+
+permissions:
+  id-token: write
+  issues: write
+  pull-requests: write
+  contents: write
+
+on:
+  push:
+    branches:
+      - v3.x/staging 
+  pull_request:
+    types: [opened, synchronize]
+  workflow_dispatch:
+
+jobs:
+  test-pswi-zowe-yaml:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: '[Prep 1] Checkout'
+        uses: actions/checkout@v4
+
+      - name: '[PSI-LOCK] Lock marist servers to build PSWI'
+        uses: zowe-actions/shared-actions/lock-resource@main
+        with:
+          lock-repository: ${{ github.repository }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          lock-resource-name: zowe-psi-build-zzow07-lock
+          lock-avg-retry-interval: 30
+
+      - name: 'Test ZWECONF workflow'
+        id: zweconf
+        timeout-minutes: 30
+        run: |
+          cd pswi
+          ./ZWECONF_test.sh
+        env:
+          ZOSMF_USER: ${{ secrets.ZOWE_PSWI_BUILD_USR }}
+          ZOSMF_PASS: ${{ secrets.ZOWE_PSWI_BUILD_PASSWD }}
+          ZZOW_SSH_PORT: ${{ secrets.SSH_MARIST_ALLSYS_PORT }}
+
+      - name: Store test output
+        uses: actions/upload-artifact@v4
+        if: steps.zweconf.outcome != '' && failure()
+        with:
+          name: zweconf_test
+          path: |
+            pswi/**.txt
+            pswi/zowe_.yaml
+            pswi/ZWECONF1/**

pswi/01_smpe.sh

@@ -91,8 +91,8 @@ STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`
 
 if [ "$STATUS_NAME" = "in-progress" ]
 then
-  echo "Workflow ended with an error."
-  echo $RESP
+  echo "Workflow with SMP/E ended with an error." >> report.txt
+  echo $RESP >> report.txt
   exit -1
 elif [ "$STATUS_NAME" = "complete" ]
 then

pswi/02_ptf.sh

@@ -87,8 +87,8 @@ STATUS_NAME=`echo $RESP | grep -o '"statusName":".*"' | cut -f4 -d\"`
 
 if [ "$STATUS_NAME" = "in-progress" ]
 then
-  echo "Workflow ended with an error."
-  echo $RESP
+  echo "Workflow with PTFs ended with an error." >> report.txt
+  echo $RESP >> report.txt
   exit -1
 elif [ "$STATUS_NAME" = "complete" ]
 then

pswi/03_create.sh

@@ -79,8 +79,8 @@ then
   then
     echo "${ZOWE_MOUNT} with zFS ${ZOWE_ZFS} mounted will be used."
   else
-    echo "The file system ${ZOWE_ZFS} exists but is mounted to different mount point ${MOUNTZ}."
-    echo "It is required to have the file system ${ZOWE_ZFS} mounted to the exact mount point (${ZOWE_MOUNT}) to successfully export Zowe PSWI."
+    echo "The file system ${ZOWE_ZFS} exists but is mounted to different mount point ${MOUNTZ}." >> report.txt
+    echo "It is required to have the file system ${ZOWE_ZFS} mounted to the exact mount point (${ZOWE_MOUNT}) to successfully export Zowe PSWI." >> report.txt
     exit -1
   fi
 else
@@ -90,7 +90,7 @@ else
   if [ -n "$MOUNTZFS" ]
   then
     # If ZFS is not mounted to the mountpoint then this ZOWE mountpoint has different zFS
-    echo "The mountpoint ${ZOWE_MOUNT} has different zFS ${MOUNTZFS}."
+    echo "The mountpoint ${ZOWE_MOUNT} has different zFS ${MOUNTZFS}." >> report.txt
     exit -1
   else
   # Mount zFS to Zowe mountpoint
@@ -108,14 +108,14 @@ RESP=`curl -s $CHECK_WORKFLOW_DSN_URL -k -X "GET" -H "Content-Type: application/
 DS_COUNT=`echo $RESP | grep -o '"returnedRows":[0-9]*' | cut -f2 -d:`
 if [ $DS_COUNT -ne 0 ]
 then
-  echo "The ${WORKFLOW_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue."
+  echo "The ${WORKFLOW_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue." >> report.txt
   exit -1 
 else
   echo "Creating a data set where the post-Deployment workflow will be stored."
   RESP=`curl -s $WORKFLOW_DSN_URL -k -X "POST" -d "$ADD_WORKFLOW_DSN_JSON" -H "Content-Type: application/json" -H "X-CSRF-ZOSMF-HEADER: A" --user $ZOSMF_USER:$ZOSMF_PASS`
   if [ -n "$RESP" ]
   then 
-    echo "The creation of the ${WORKFLOW_DSN} was not successful. Error message: ${RESP}"
+    echo "The creation of the ${WORKFLOW_DSN} was not successful. Error message: ${RESP}" >> report.txt
     exit -1
   fi  
 fi
@@ -173,7 +173,7 @@ RESP=`curl -s $CHECK_EXPORT_DSN_URL -k -X "GET" -H "Content-Type: application/js
 DSN_COUNT=`echo $RESP | grep -o '"returnedRows":[0-9]*' | cut -f2 -d:`
 if [ $DSN_COUNT -ne 0 ]
 then
-  echo "The ${EXPORT_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue."
+  echo "The ${EXPORT_DSN} already exist. Because there is a possibility that it contains something unwanted the script does not continue." >> report.txt
   exit -1
 else
   echo "Creating a data set where the export jobs will be stored."
@@ -209,7 +209,7 @@ if [ $? -gt 0 ];then exit -1;fi
 LOAD_STATUS_URL=`echo $RESP | grep -o '"statusurl":".*"' | cut -f4 -d\" | tr -d '\' 2>/dev/null`
 if [ -z "$LOAD_STATUS_URL" ]
 then
-  echo "No response from the REST API call."
+  echo "No response from the load product REST API call." >> report.txt
   exit -1
 fi
 
@@ -239,7 +239,7 @@ if [ $? -gt 0 ];then exit -1;fi
 EXPORT_STATUS_URL=`echo $RESP | grep -o '"statusurl":".*"' | cut -f4 -d\" | tr -d '\' 2>/dev/null`
 if [ -z "$EXPORT_STATUS_URL" ]
 then
-  echo "No response from the REST API call."
+  echo "No response from the export REST API call." >> report.txt
   exit -1
 fi
 
@@ -266,7 +266,7 @@ then
   # Can be 100% but still running
   if [ "$STATUS" != "complete" ] && [ "$STATUS" != "running" ]
   then
-    echo "Status of generation of Export JCL failed."
+    echo "Status of generation of Export JCL failed." >> report.txt
     exit -1
   fi
 fi
@@ -275,7 +275,7 @@ done
 
 if [ -z "$DSN" ]
 then
-  echo "The creation of export JCL failed"
+  echo "The creation of export JCL failed" >> report.txt
   exit -1
 fi
 

pswi/051_test_workflows.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+#version=1.0
+
+export BASE_URL="${ZOSMF_URL}:${ZOSMF_PORT}"
+
+echo ""
+echo ""
+echo "Script for testing workflows..."
+echo "Host                   :" $ZOSMF_URL
+echo "Port                   :" $ZOSMF_PORT
+echo "SSH Port               :" $ZZOW_SSH_PORT
+echo "z/OSMF system          :" $ZOSMF_SYSTEM
+echo "Test HLQ               :" $TEST_HLQ
+echo "Test mount point       :" $TEST_MOUNT
+echo "Work zFS               :" $WORK_ZFS   # For z/OSMF v2.3
+echo "Work mount point       :" $WORK_MOUNT # For z/OSMF v2.3
+
+echo "Mounting ${TEST_HLQ}.ZFS"
+sh scripts/tmp_mounts.sh "${TEST_HLQ}.ZFS" "${TEST_MOUNT}"
+if [ $? -gt 0 ]; then exit -1; fi
+
+echo "Registering/testing the configuration workflow ${TEST_HLQ}.WORKFLOW(ZWECONF)"
+sh scripts/wf_run_test.sh "${TEST_HLQ}.WORKFLOW(ZWECONF)"
+if [ $? -gt 0 ];then exit -1;fi
+
+echo "Registering/testing the configuration workflow ${TEST_MOUNT}/files/workflows/ZWECONF.xml"
+sh scripts/wf_run_test.sh "${TEST_MOUNT}/files/workflows/ZWECONF.xml"
+if [ $? -gt 0 ];then exit -1;fi

pswi/05_test.sh

@@ -72,15 +72,3 @@ echo " Running the deployment test for z/OSMF version 2.3"
 
 pip install requests
 python scripts/deploy_test_2_3.py
-
-echo "Mounting ${TEST_HLQ}.ZFS"
-sh scripts/tmp_mounts.sh "${TEST_HLQ}.ZFS" "${TEST_MOUNT}"
-if [ $? -gt 0 ]; then exit -1; fi
-
-echo "Registering/testing the configuration workflow ${TEST_HLQ}.WORKFLOW(ZWECONF)"
-sh scripts/wf_run_test.sh "${TEST_HLQ}.WORKFLOW(ZWECONF)"
-if [ $? -gt 0 ];then exit -1;fi
-
-echo "Registering/testing the configuration workflow ${TEST_MOUNT}/content/files/workflows/ZWECONF.xml"
-sh scripts/wf_run_test.sh "${TEST_MOUNT}/files/workflows/ZWECONF.xml"
-if [ $? -gt 0 ];then exit -1;fi

pswi/PSWI-marist.sh

@@ -1,12 +1,13 @@
+set -x
 export ZOSMF_URL="https://zzow07.zowe.marist.cloud"
 export ZOSMF_PORT=10443
 export ZOSMF_SYSTEM="S0W1"
-export DIR="/u/zowead2"
-export SMPEHLQ="ZOWEAD2"
-export TMP_ZFS="ZOWEAD2.TMP.ZFS"
+export DIR="/u/${ZOSMF_USER,,}"
+export SMPEHLQ="${ZOSMF_USER}"
+export TMP_ZFS="${ZOSMF_USER}.TMP.ZFS"
 export ZOWE_MOUNT="/u/zwe/zowe-smpe/"
 export VOLUME="ZOS003"
-export TEST_HLQ="ZOWEAD2.PSWIT"
+export TEST_HLQ="${ZOSMF_USER}.PSWIT"
 export SYSAFF="(S0W1)"
 export ACCOUNT=1
 
@@ -29,7 +30,7 @@ export TMP_MOUNT="${DIR}/zowe-tmp"
 export TEST_MOUNT="${DIR}/test_mount"
 export EXPORT="${TMP_MOUNT}/export/"
 export WORK_MOUNT="${DIR}/work"
-export WORK_ZFS="ZOWEAD2.WORK.ZFS"
+export WORK_ZFS="${ZOSMF_USER}.WORK.ZFS"
 export ZOSMF_V="2.3"
 export SMPE_WF_NAME="ZOWE_SMPE_WF"
 export PTF_WF_NAME="ZOWE_PTF_WF"
@@ -41,7 +42,7 @@ if [ -f ../.pax/zowe-smpe.zip ]; then
   mkdir -p "unzipped"
   unzip ../.pax/zowe-smpe.zip -d unzipped
 else
-  echo "zowe-smpe file not found"
+  echo "zowe-smpe file not found" >> report.txt
   exit -1
 fi
 
@@ -73,7 +74,7 @@ else
   if [ -f ../.pax/${FMID}.zip ]; then
     unzip ../.pax/${FMID}.zip -d unzipped
   else
-    echo "File with FMID not found"
+    echo "File with FMID not found" >> report.txt
     exit -1
   fi
 fi
@@ -90,6 +91,14 @@ export WORKFLOW_DSN=${CSIHLQ}.WORKFLOW
 export ZOWE_ZFS="${CSIHLQ}.ZFS"
 export VERSION=$(cat ../manifest.json.template | grep -o '"version": ".*"' | head -1 | cut -f4 -d\")
 
+# Initialize variables
+presmpe=0
+smpe=0
+ptf=0
+create=0
+test=0
+wf_test=0
+
 # Upload and prepare all files
 sh 00_presmpe.sh
 presmpe=$?
@@ -125,7 +134,13 @@ if [ $presmpe -eq 0 ]; then
         # Test PSWI
         sh 05_test.sh
         test=$?
-
+
+        if [ $test -eq 0 ]; then
+          #test the workflows
+          sh 051_test_workflows.sh
+          wf_test=$?
+        fi
+
         # Cleanup after the test
         sh 06_test_cleanup.sh
       fi
@@ -141,23 +156,28 @@ if [ $presmpe -eq 0 ]; then
     # Clean RELFILEs and PTFs
     sh 08_presmpe_cleanup.sh
   fi
+else
+  # Clean RELFILEs and PTFs
+  sh 08_presmpe_cleanup.sh
 fi
 
 echo ""
 echo ""
 
-if [ $smpe -ne 0 ] || [ $ptf -ne 0 ] || [ $create -ne 0 ] || [ $test -ne 0 ] || [ $presmpe -ne 0 ]; then
-  echo "Build unsuccessful!"
+if [ $smpe -ne 0 ] || [ $ptf -ne 0 ] || [ $create -ne 0 ] || [ $test -ne 0 ] || [ $presmpe -ne 0 ] || [ $wf_test -ne 0 ]; then
+  echo "Build unsuccessful!" >> report.txt
   if [ $presmpe -ne 0 ]; then
-    echo "Pre-SMP/E wasn't successful."
+    echo "Pre-SMP/E wasn't successful." >> report.txt
   elif [ $smpe -ne 0 ]; then
-    echo "SMP/E wasn't successful."
+    echo "SMP/E wasn't successful." >> report.txt
   elif [ $ptf -ne 0 ]; then
-    echo "Applying PTFs wasn't successful."
+    echo "Applying PTFs wasn't successful." >> report.txt
   elif [ $create -ne 0 ]; then
-    echo "Creation of PSWI wasn't successful."
+    echo "Creation of PSWI wasn't successful." >> report.txt
   elif [ $test -ne 0 ]; then
-    echo "Testing of PSWI wasn't successful."
+    echo "Testing of PSWI wasn't successful." >> report.txt
+  elif [ $wf_test -ne 0 ]; then
+    echo "Workflow testing wasn't successful." >> report.txt
   fi
   exit -1
 else

pswi/ZWECONF_test.sh

@@ -0,0 +1,60 @@
+export ZOSMF_URL="https://zzow07.zowe.marist.cloud"
+export ZOSMF_PORT=10443
+export ZOSMF_SYSTEM="S0W1"
+export JOBNAME="ZWECONF1"
+export HOST=${ZOSMF_URL#https:\/\/}
+export BASE_URL="${ZOSMF_URL}:${ZOSMF_PORT}"
+WORK_MOUNT="/tmp"
+
+echo "Changing runtime path in ZWECONF.properties."
+
+cp ../workflows/files/ZWECONF.properties ./ZWECONF.properties
+sed "s|runtimeDirectory=|runtimeDirectory=${WORK_MOUNT}|g" ./ZWECONF.properties > _ZWECONF
+sed "s|java_home=|java_home=#delete_me#|g" _ZWECONF > ZWECONF
+sed "s|node_home=|node_home=#delete_me#|g" ZWECONF > _ZWECONF
+
+echo "Changing the configuration workflow to be fully automated."
+
+cp ../workflows/files/ZWECONF.xml ./ZWECONF.xml
+sed "s|<autoEnable>false|<autoEnable>true|g" ./ZWECONF.xml > ZWECONFX
+
+sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
+cd ${WORK_MOUNT}
+put _ZWECONF
+put ZWECONFX
+EOF
+
+echo "Testing the configuration workflow ${WORK_MOUNT}/ZWECONFX"
+sh scripts/wf_run_test.sh "${WORK_MOUNT}/ZWECONFX" "run" "ZWECONF" "${WORK_MOUNT}/_ZWECONF"
+if [ $? -gt 0 ];then exit -1;fi
+
+echo "Converting zowe.yaml"
+
+echo "//${ZOSMF_SYSTEM} JOB (1),'PSWI',MSGCLASS=A,REGION=0M" > JCL
+echo "/*JOBPARM SYSAFF=(${ZOSMF_SYSTEM})" >> JCL
+echo "//UNPAXDIR EXEC PGM=BPXBATCH" >> JCL
+echo "//STDOUT DD SYSOUT=*" >> JCL
+echo "//STDERR DD SYSOUT=*" >> JCL
+echo "//STDPARM  DD *" >> JCL
+echo "SH set -x;set -e;" >> JCL
+echo "cd ${WORK_MOUNT};" >> JCL
+echo "iconv -f IBM-1047 -t ISO8859-1 zowe.yaml > zowe_.yaml;" >> JCL
+echo "/*" >> JCL
+
+sh scripts/submit_jcl.sh "`cat JCL`"
+if [ $? -gt 0 ];then exit -1;fi
+rm JCL
+
+sshpass -p${ZOSMF_PASS} sftp -o HostKeyAlgorithms=+ssh-rsa -o BatchMode=no -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -b - -P ${ZZOW_SSH_PORT} ${ZOSMF_USER}@${HOST} << EOF
+cd ${WORK_MOUNT}
+get zowe_.yaml
+rm zowe_.yaml
+EOF
+
+pwd
+
+cp ../example-zowe.yaml example-zowe.yaml
+
+diff --ed example-zowe.yaml zowe_.yaml > diff.txt || true
+
+diff --ed diff.txt scripts/base_diff.txt > final_diff.txt