Store quality gate state on failure (#364)

* stash qg state on run

Signed-off-by: Alex Goodman <[email protected]>

* update instructions

Signed-off-by: Alex Goodman <[email protected]>

---------

Signed-off-by: Alex Goodman <[email protected]>

.github/actions/quality-gate/action.yaml

@@ -15,7 +15,47 @@ runs:
       working-directory: tests/quality
       run: poetry run make capture provider=${{ inputs.provider }}
 
+
     - name: Validate provider results
       shell: bash
       working-directory: tests/quality
       run: poetry run make validate
+
+    - name: Archive the provider state (${{ inputs.provider }})
+      if: ${{ failure() }}
+      shell: bash
+      run: tar -czvf qg-capture-state-${{ inputs.provider }}.tar.gz -C tests/quality --exclude tools --exclude labels .yardstick.yaml .yardstick
+
+    - name: Upload the provider state archive (${{ inputs.provider }})
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: qg-capture-state-${{ inputs.provider }}
+        path: qg-capture-state-${{ inputs.provider }}.tar.gz
+
+    - name: Show instructions to debug
+      if: ${{ failure() }}
+      shell: bash
+      run: |
+        ARCHIVE_BASENAME=qg-capture-state-${{ inputs.provider }}
+        ARCHIVE_NAME=$ARCHIVE_BASENAME.zip
+
+        cat << EOF >> $GITHUB_STEP_SUMMARY
+        ## Troubleshooting '${{ inputs.provider }}' failed run
+
+        Download the artifact from this workflow run: \`$ARCHIVE_NAME\`
+
+        Then run the following commands to debug:
+        \`\`\`bash
+        # copy the archive to the tests/quality directory
+        cd tests/quality
+        unzip $ARCHIVE_NAME && tar -xzf $ARCHIVE_BASENAME.tar.gz
+        \`\`\`
+
+        Now you can debug the provider with yardstick:
+        \`\`\`bash
+        poetry shell
+        yardstick result list
+        yardstick label explore
+        \`\`\`
+        EOF

.gitignore

@@ -4,6 +4,7 @@
 .grype
 .grype-db
 *.tar.gz
+*.zip
 
 /bin
 /data/