Add scripts to analyze test results

.gitignore

@@ -1,4 +1,3 @@
-changes.json
 test-data
 test.sh
 .vscode

README.md

@@ -0,0 +1,5 @@
+# openpgp-interop-test-analyzer
+
+Scripts to analyze test results from the Openpgp interoperability test suite.
+
+See https://gitlab.com/sequoia-pgp/openpgp-interoperability-test-suite

action.yml

@@ -0,0 +1,79 @@
+name: 'Compare with baseline'
+description: 'Analyze the results of the interoperability test suite'
+
+inputs:
+
+  baseline:
+    description: 'The id of the baseline implementation'
+    required: true
+
+  target:
+    description: 'The id of the target implementation'
+    required: true
+
+  results: 
+    description: 'Path to the test results'
+    required: true
+    default: 'test-suite-results.json'
+
+  output:
+    description: 'Path to the output'
+    required: true
+    default: 'baseline_comparison.json'
+
+  summarize:
+    description: 'Whether to produce a small summary for the job'
+    required: true
+    default: 'true'
+
+  fail-on-regression:
+    description: 'Whether to fail if a regression was found'
+    required: true
+    default: 'true'
+
+  upload:
+    description: 'Whether to upload the comparison results as an artifact'
+    required: true
+    default: 'true'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repo
+      uses: actions/checkout@v3
+      with:
+        repository: marinthiercelin/sop-test-analyzer
+        path: analyzer
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Show files
+      run: ls -la .
+      shell: bash
+    - name: Compare with baseline
+      run: cat $RESULTS | python analyzer/clean_up_results.py | python analyzer/compare_with_baseline.py $BASELINE $TARGET > $OUTPUT
+      shell: bash
+      env:
+        RESULTS: ${{ inputs.results }}
+        BASELINE: ${{ inputs.baseline }}
+        TARGET: ${{ inputs.target }}
+        OUTPUT: ${{ inputs.output }}
+    - if: ${{ inputs.upload == 'true' }}
+      name: Upload comparison with main
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.output }}
+        path: ${{ inputs.output }}
+    - if: ${{ inputs.summarize == 'true' }}
+      name: Summarize test results
+      run: cat $OUTPUT | python analyzer/summarize.py >> $GITHUB_STEP_SUMMARY
+      shell: bash
+      env:
+        OUTPUT: ${{ inputs.output }}
+    - if: ${{ inputs.fail-on-regression == 'true' }}
+      name: "Fail action on regression"
+      run: cat $OUTPUT | python analyzer/pass_or_fail.py
+      shell: bash
+      env:
+        OUTPUT: ${{ inputs.output }}

compare_with_baseline.py

@@ -7,6 +7,7 @@ def main():
     (baseline, target) = get_args()
     all_changes = compare_with_baseline(data, baseline, target)
     json.dump(all_changes, sys.stdout, indent=2)
+
 
 def compare_with_baseline(data, baseline, target):
     all_changes = []

pass_or_fail.py

@@ -0,0 +1,20 @@
+import sys
+import json
+
+def main():
+    data = json.load(sys.stdin)
+    if has_regression(data) :
+        sys.exit("Found a regression")
+
+REGRESSION = "Regression"
+
+def has_regression(data):
+    for section in data:
+        for test in section["changes"] :
+            for run in test["changes"] :
+                if run["change"] == REGRESSION:
+                    return True
+    return False
+
+if __name__ == "__main__":
+    main()

summarize.py

@@ -0,0 +1,34 @@
+import sys
+import json
+
+def main():
+    data = json.load(sys.stdin)
+    summarize(data)
+
+REGRESSION = "Regression"
+IMPROVEMENT = "Improvement"
+
+def summarize(data):
+    regression = 0
+    improvement = 0
+    for section in data:
+        for test in section["changes"] :
+            for run in test["changes"] :
+                if run["change"] == REGRESSION:
+                    regression += 1
+                elif run["change"] == IMPROVEMENT:
+                    improvement += 1
+                else:
+                    print("Unknown change type %s" % run["change"], file=sys.stderr)
+    regression_summary = plural(regression, "regression")
+    improvement_summary = plural(improvement, "improvement")
+    summary = "Summary: {}, {}".format(regression_summary, improvement_summary)
+    print(summary)
+
+def plural(value, name):
+    if value != 1:
+        name += "s"
+    return "{} {}".format(value, name)
+
+if __name__ == "__main__":
+    main()