test: Audit some integration tests, make minor changes, and add plain…

…text test list (#2089) * Add script to summarize integration tests * Add error messages for missing docstrings * Use better separator * Add docstring to test * Add some docstrings and adjust formatting * Add some more docstrings * Add makefile and git workflow * Add else case * Use temporary files instead of substitution * Add missing fi * Adding a docstring without regenerating the test docs * Give testing-docs workflow write permission * Specify branch to push to * Just fail the test instead of pushing automatically * Regenerate testing docs * Rename workflow * Unify type imports * Add newline around fenced codeblock * Fix repeated 'or' * Regenerate testing doc * Expand docstring for TestAllocateTokens * Adjust testing docs workflow * Improved some test descriptions * Improved misbehavior.go test descriptions * Improved normal_operations.go test descriptions * Improved provider_gov_hooks.go test descriptions * Improved query_providerinfo_test.go test descriptions * Removed warnings from setup.go * Improved slashing.go test descriptions * Improved stop_consumer.go test descriptions * Improved throttle_retry.go test descriptions * Improved throttle.go test descriptions * Fixed spelling mistake * Improved valset_update.go test descriptions * Improved test documentation * Improved testing-docs.yml * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Update testing documentation * Testing git workflow for testing-docs * Testing git workflow for testing-docs * Update testing documentation * Testing git workflow for testing-docs * Update testing documentation * Improved some test descriptions * Update testing documentation * Fix minor typos * Finished testing-docs.yml * Fix minor typos * Rename test from testing-docs.yml * Cleaned up TODOs and updated test descriptions * Added check in misbehaviour.go * Fixed integration tests * Removed additional comments from key_assignment.go * Removed outdated file descriptions * Update testing documentation * Update .github/workflows/testing-docs.yml Co-authored-by: Marius Poke <[email protected]> * Update tests/integration/README.md * Removed TestRelayAndApplyDoubleSignPacket test from slashing.go * Update testing documentation * Update .github/workflows/testing-docs.yml Co-authored-by: Marius Poke <[email protected]> * Update scripts/test_doc/test_documentation.md Co-authored-by: Marius Poke <[email protected]> * Update testing documentation * Integration test comments refactored * Update testing documentation * Update .github/workflows/testing-docs.yml Co-authored-by: Marius Poke <[email protected]> * Changed inconsistent make command name * Testing testing-docs script * Update testing documentation * Testing testing-docs script * Update testing documentation * Changed inconsistent make command in TESTING.md * Updated testing-docs.yml * Updated testing-docs.yml * Testing testing-docs.yml * Update testing documentation * Testing testing-docs.yml * Update testing documentation * Removed failiure signal from testing-docs.yml * Removed TestValidatorDoubleSigning from slashing.go * Added TODOs in slashing.go * Added panics and a TODO for unexpected errors in key_assignment.go. * Implement security check for file path in extract_docstrings.go * Removed security check for file path in extract_docstrings.go --------- Co-authored-by: kirdatatjana <[email protected]> Co-authored-by: github-actions <[email protected]> Co-authored-by: kirdatatjana <[email protected]> Co-authored-by: Marius Poke <[email protected]>
cosmos · Oct 10, 2024 · acbc4c7 · acbc4c7
1 parent 47cd04e
commit acbc4c7
Show file tree

Hide file tree

Showing 24 changed files with 766 additions and 226 deletions.
diff --git a/.github/workflows/testing-docs.yml b/.github/workflows/testing-docs.yml
@@ -0,0 +1,70 @@
+name: Generate testing docs
+
+on:
+    workflow_call:
+    pull_request:
+    merge_group:
+    push:
+      branches:
+        - main
+        - release/v*
+        - feat/*
+
+permissions:
+  contents: write
+  pull-requests: write      # 'write' access to pull requests in order to update test documentation automatically 
+
+jobs:
+    testing-docs:
+        runs-on: ubuntu-latest
+
+        steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+
+        - uses: actions/setup-go@v5
+          with:
+            go-version: "1.22"
+            check-latest: true
+            cache: true
+            cache-dependency-path: go.sum
+        - uses: technote-space/[email protected]
+          id: git_diff
+          with:
+            PATTERNS: |
+              tests/integration/**/*.go
+              **/Makefile
+              Makefile
+
+        - name: Generate testing docs
+          if: env.GIT_DIFF
+          run: make build-testing-docs
+
+        - name: Check for changes and update automatically
+          if: env.GIT_DIFF
+          id: check_changes_and_update
+          run: |
+            git show HEAD:scripts/test_doc/test_documentation.md > committed_file.md
+            cp scripts/test_doc/test_documentation.md generated_file.md
+            if ! diff -q generated_file.md committed_file.md; then
+              echo "Documentation for integration tests is out of date. Updating and pushing changes..."
+              cp generated_file.md scripts/test_doc/test_documentation.md
+
+              if [ -n "$GITHUB_HEAD_REF" ]; then
+                branch=$GITHUB_HEAD_REF
+              else
+                branch=${GITHUB_REF#refs/heads/}
+              fi
+              
+              git fetch origin $branch
+              git checkout $branch
+
+              cp generated_file.md scripts/test_doc/test_documentation.md
+              git config user.name "github-actions"
+              git config user.email "[email protected]"
+              git add scripts/test_doc/test_documentation.md
+              git commit -m "Update testing documentation"
+
+              git push origin "$branch"
+
+            fi
diff --git a/Makefile b/Makefile
@@ -265,6 +265,9 @@ build-docs-deploy:
 build-docs-local:
 	@cd docs && ./build_local.sh
 
+build-testing-docs:
+	@cd scripts/test_doc && go run extract_docstrings.go
+
 ###############################################################################
 ### 							Test Traces									###
 ###############################################################################

diff --git a/TESTING.md b/TESTING.md
@@ -14,6 +14,18 @@ Unit tests are useful for simple standalone functionality, and CRUD operations.
 
 To run integration tests against your own consumer/provider implementations, use [instance_test.go](tests/integration/instance_test.go) as an example. All you'll need to do is make sure your applications implement the necessary interfaces defined in [interfaces.go](testutil/integration/interfaces.go), pattern match [specific_setup.go](testutil/ibc_testing/specific_setup.go), then pass in the appropriate types and parameters to the suite, as is done in `instance_test.go` for the dummy provider/consumer implementations.
 
+A list of test scenarios covered by integration tests can be found in [scripts/test_doc/test_documentation.md](scripts/test_doc/test_documentation.md).
+When adding an integration test, write a brief description as a docstring in the Golang code in this schema:
+
+```go
+// This is a test that tests foo and bar.
+// @Long Description@
+// Here is a detailed description
+// that goes into more detail and describes the scenario.
+```
+
+Then, run `make build-testing-docs` to update the test documentation.
+
 ## Model-Based Tests (MBT)
 
 [MBT](tests/mbt/) tests are similar to integration tests, but they compare the system state to an expected state generated from a formally verified specification written in Quint.

diff --git a/scripts/test_doc/extract_docstrings.go b/scripts/test_doc/extract_docstrings.go
@@ -0,0 +1,139 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	// Define the directory to traverse and the output markdown file
+	dir := "../../tests/integration"
+	outputFile := "test_documentation.md"
+
+	// Open the output file
+	out, err := os.Create(outputFile)
+	if err != nil {
+		log.Fatalf("Error creating output file: %v\n", err)
+	}
+	defer out.Close()
+
+	// Write the header for the markdown file
+	fmt.Fprintf(out, "# Test Documentation\n\n")
+
+	errorStatusCode := false
+
+	// Walk through the directory
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Only process .go files
+		if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
+			functionsMissingDocStrings := extractDocstrings(path, out)
+			if len(functionsMissingDocStrings) > 0 {
+				fmt.Printf("The following test functions in %s are missing docstrings:\n", path)
+				for _, fn := range functionsMissingDocStrings {
+					fmt.Printf("\t%s\n", fn)
+				}
+				errorStatusCode = true
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Fatalf("Error walking the path %q: %v\n", dir, err)
+	}
+
+	fmt.Printf("Documentation generated successfully in %s\n", outputFile)
+
+	if errorStatusCode {
+		os.Exit(1)
+	}
+}
+
+// extractDocstrings extracts the docstrings from the Go source file and writes them to the output file
+// in a markdown table format.
+// It returns a list of test functions that are missing docstrings.
+func extractDocstrings(filePath string, out *os.File) []string {
+	// Read the Go source file
+	src, err := os.ReadFile(filePath) // #nosec G304
+	if err != nil {
+		log.Fatalf("Error reading file %s: %v\n", filePath, err)
+	}
+
+	// Create the AST for the source file
+	fset := token.NewFileSet()
+	node, err := parser.ParseFile(fset, filePath, src, parser.ParseComments)
+	if err != nil {
+		log.Fatalf("Error parsing file %s: %v\n", filePath, err)
+	}
+
+	functionsMissingDocstrings := []string{}
+
+	// Files that do not contain test functions are excluded from the documentation.
+	fileNameWritten := false
+
+	// Traverse the AST
+	for _, f := range node.Decls {
+		if fn, isFn := f.(*ast.FuncDecl); isFn && strings.HasPrefix(fn.Name.Name, "Test") {
+			// Check if the function has a docstring
+			if fn.Doc != nil {
+
+				if !fileNameWritten {
+					relativePath := strings.TrimPrefix(filePath, "../../tests/integration/")
+					doclink := fmt.Sprintf("[%s](%s)", relativePath, filePath)
+					fmt.Fprintf(out, "# %s \n", doclink)
+					fmt.Fprintf(out, "<details><summary> Test Specifications </summary>\n\n")
+
+					// Write table header
+					fmt.Fprintf(out, "| Function | Short Description |\n")
+					fmt.Fprintf(out, "|----------|-------------------|\n")
+					fileNameWritten = true
+				}
+
+				doc := fn.Doc.Text()
+				link := fmt.Sprintf("[%s](%s#L%d)", fn.Name.Name, filePath, fset.Position(fn.Pos()).Line)
+
+				// Split the docstring based on the separator "========"
+				parts := strings.Split(doc, "\n@Long Description@\n")
+				var shortDescription, longDescription string
+				if len(parts) > 1 {
+					shortDescription = strings.TrimSpace(parts[0])
+					longDescription = strings.TrimSpace(parts[1])
+				} else {
+					shortDescription = strings.TrimSpace(doc)
+					longDescription = ""
+				}
+
+				// Format the description
+
+				// avoid breaking the table format: newlines need to be replaced
+				// for the short description, use spaces
+				shortDescription = strings.ReplaceAll(shortDescription, "\n", " ")
+				// for the long description, use breaks
+				longDescription = strings.ReplaceAll(longDescription, "\n", "<br>")
+
+				description := shortDescription
+				if longDescription != "" {
+					description += fmt.Sprintf("<details><summary>Details</summary>%s</details>", longDescription)
+				}
+
+				fmt.Fprintf(out, " %s | %s |\n", link, description)
+			} else {
+				functionsMissingDocstrings = append(functionsMissingDocstrings, fn.Name.Name)
+
+			}
+		}
+	}
+	if fileNameWritten {
+		fmt.Fprintf(out, "</details>\n\n")
+	}
+	return functionsMissingDocstrings
+}