diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 22968f1531b4..99cbd66265cc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -111,7 +111,7 @@ Prerequisites: - [ ] Test 2 ### Test Coverage - + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 18aa94ff1059..f29fa7b2cfaa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,7 +57,7 @@ updates: # Check for version updates for Python dependencies (coverage) - package-ecosystem: "pip" - directory: "/supporting_scripts/generate_code_cov_table" + directory: "/supporting_scripts/code-coverage/generate_code_cov_table" schedule: interval: "weekly" reviewers: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ccdd88feed03..0155d5d63248 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,32 @@ jobs: ${{ env.java }} cache: 'gradle' - name: Java Tests - run: set -o pipefail && ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log + run: | + set -o pipefail + + DEFAULT_BRANCH="${{ github.event.repository.default_branch }}" + CURRENT_BRANCH="${{ github.ref_name }}" + + if [[ "$DEFAULT_BRANCH" != "$CURRENT_BRANCH" ]]; then + # Explicitly fetch as the clone action only clones the current branch + git fetch origin "$DEFAULT_BRANCH" + + chmod +x ./supporting_scripts/get_changed_modules.sh + CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH") + + # Restrict executed tests to changed modules if there is diff between this and the base branch + if [ -n "${CHANGED_MODULES}" ]; then + IFS=, + TEST_MODULE_TAGS=$(echo "-DincludeModules=${CHANGED_MODULES[*]}") + + echo "Executing tests for modules: $CHANGED_MODULES" + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log + exit 0 + fi + fi + + echo "Executing all tests" + ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log - name: Print failed tests if: failure() run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests." @@ -92,8 +117,13 @@ jobs: uses: actions/upload-artifact@v4 with: name: Coverage Report Server Tests - path: build/reports/jacoco/test/html/ - + path: build/reports/jacoco/test/html + - name: Append Per-Module Coverage to Job Summary + if: success() || failure() + run: | + AGGREGATED_REPORT_FILE=./module_coverage_report.md + python3 ./supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py build/reports/jacoco $AGGREGATED_REPORT_FILE + cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY server-tests-mysql: needs: [ server-tests ] diff --git a/build.gradle b/build.gradle index 27099c72db3d..923fea2789ac 100644 --- a/build.gradle +++ b/build.gradle @@ -118,16 +118,38 @@ modernizer { exclusions = ["java/util/Optional.get:()Ljava/lang/Object;"] } +// Allow using in jacoco.gradle +ext { + includedTestTags = System.getProperty("includeTags") + includedTags = !includedTestTags ? new String[]{} : includedTestTags.split(",") as String[] + includedModulesTag = System.getProperty("includeModules") + includedModules = !includedModulesTag ? new String[]{} : includedModulesTag.split(",") as String[] + + runAllTests = includedTags.size() == 0 && includedModules.size() == 0 + BasePath = "de/tum/cit/aet/artemis" +} + // Execute the test cases: ./gradlew test // Execute only architecture tests: ./gradlew test -DincludeTags="ArchitectureTest" +// Execute tests only for specific modules: ./gradlew test -DincludeModules="atlas". Using this flag, "includeTags" will be ignored. test { - if (System.getProperty("includeTags")) { - useJUnitPlatform { - includeTags System.getProperty("includeTags") + if (runAllTests) { + useJUnitPlatform() + exclude "**/*IT*", "**/*IntTest*" + } else if (includedModules.size() == 0) { + // not running all tests, but not module-specific ones -> use tags + useJUnitPlatform() { + includeTags includedTags } } else { useJUnitPlatform() - exclude "**/*IT*", "**/*IntTest*" + // Always execute "shared"-folder when executing module-specifc tests + includedModules += "shared" + filter { testFilter -> + includedModules.each { val -> + testFilter.includeTestsMatching("de.tum.cit.aet.artemis.$val.*") + } + } } testLogging { @@ -144,59 +166,12 @@ tasks.register("testReport", TestReport) { testResults.from(test) } -jacoco { - toolVersion = "0.8.12" -} - jar { enabled = false } -private excludedClassFilesForReport(classDirectories) { - classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, - exclude: [ - "**/de/tum/cit/aet/artemis/**/domain/**/*_*", - "**/de/tum/cit/aet/artemis/core/config/migration/entries/**", - "**/gradle-wrapper.jar/**" - ] - ) - })) -} - -jacocoTestReport { - reports { - xml.required = true - } - // we want to ignore some generated files in the domain folders - afterEvaluate { - excludedClassFilesForReport(classDirectories) - } -} - -jacocoTestCoverageVerification { - violationRules { - rule { - limit { - counter = "INSTRUCTION" - value = "COVEREDRATIO" - // TODO: in the future the following value should become higher than 0.92 - minimum = 0.892 - } - limit { - counter = "CLASS" - value = "MISSEDCOUNT" - // TODO: in the future the following value should become less than 10 - maximum = 65 - } - } - } - // we want to ignore some generated files in the domain folders - afterEvaluate { - excludedClassFilesForReport(classDirectories) - } -} -check.dependsOn jacocoTestCoverageVerification +// Dynamic generation of jacoco test report generation-/coverage verification-tasks (per-module) +apply from: "gradle/jacoco.gradle" configurations { providedRuntime diff --git a/gradle/jacoco.gradle b/gradle/jacoco.gradle new file mode 100644 index 000000000000..7100591661fd --- /dev/null +++ b/gradle/jacoco.gradle @@ -0,0 +1,217 @@ +ext { + AggregatedCoverageThresholds = [ + "INSTRUCTION": 0.900, + "CLASS": 10 + ]; + // (Isolated) thresholds when executing each module on its own + ModuleCoverageThresholds = [ + "assessment" : [ + "INSTRUCTION": 0.774, + "CLASS": 9 + ], + "athena" : [ + "INSTRUCTION": 0.863, + "CLASS": 2 + ], + "atlas" : [ + "INSTRUCTION": 0.846, + "CLASS": 13 + ], + "buildagent" : [ + "INSTRUCTION": 0.304, + "CLASS": 13 + ], + "communication": [ + "INSTRUCTION": 0.893, + "CLASS": 7 + ], + "core" : [ + "INSTRUCTION": 0.658, + "CLASS": 69 + ], + "exam" : [ + "INSTRUCTION": 0.914, + "CLASS": 1 + ], + "exercise" : [ + "INSTRUCTION": 0.649, + "CLASS": 9 + ], + "fileupload" : [ + "INSTRUCTION": 0.944, + "CLASS": 0 + ], + "iris" : [ + "INSTRUCTION": 0.706, + "CLASS": 22 + ], + "lecture" : [ + "INSTRUCTION": 0.867, + "CLASS": 0 + ], + "lti" : [ + "INSTRUCTION": 0.770, + "CLASS": 3 + ], + "modeling" : [ + "INSTRUCTION": 0.892, + "CLASS": 2 + ], + "plagiarism" : [ + "INSTRUCTION": 0.770, + "CLASS": 1 + ], + "programming" : [ + "INSTRUCTION": 0.861, + "CLASS": 13 + ], + "quiz" : [ + "INSTRUCTION": 0.785, + "CLASS": 6 + ], + "text" : [ + "INSTRUCTION": 0.847, + "CLASS": 0 + ], + "tutorialgroup": [ + "INSTRUCTION": 0.915, + "CLASS": 0 + ], + ] + // If no explicit modules defined -> generate reports and validate for each module + reportedModules = includedModules.size() == 0 + ? ModuleCoverageThresholds.collect {element -> element.key} + : includedModules as ArrayList + + // we want to ignore some generated files in the domain folders + ignoredDirectories = [ + "**/$BasePath/**/domain/**/*_*", + "**/$BasePath/core/config/migration/entries/**", + "**/gradle-wrapper.jar/**" + ] +} + +jacoco { + toolVersion = "0.8.12" +} + +jacocoTestReport { + // For the aggregated report + reports { + xml.required = true + xml.outputLocation = file("build/reports/jacoco/test/jacocoTestReport.xml") + html.required = true + html.outputLocation = file("build/reports/jacoco/test/html") + } + + finalizedBy reportedModules + .collect { module -> registerJacocoReportTask(module as String, jacocoTestReport) } + .findAll { task -> task != null} +} + +jacocoTestCoverageVerification { + // Only run full coverage when no specific modules set + enabled = reportedModules.size() == 0 + + def minInstructionCoveredRatio = AggregatedCoverageThresholds["INSTRUCTION"] as double + def maxNumberUncoveredClasses = AggregatedCoverageThresholds["CLASS"] as int + applyVerificationRule(jacocoTestCoverageVerification, minInstructionCoveredRatio, maxNumberUncoveredClasses) + + finalizedBy reportedModules + .collect { module -> registerJacocoTestCoverageVerification(module as String, jacocoTestCoverageVerification) } + .findAll { task -> task != null} +} +check.dependsOn jacocoTestCoverageVerification + +/** + * Registers a JacocoReport task based on the provided parameters. + * + * @param moduleName The module name to include in the report. + * @param rootTask The root JacocoReport root task. + * @return The configured JacocoReport task. + */ +private JacocoReport registerJacocoReportTask(String moduleName, JacocoReport rootTask) { + def taskName = "jacocoCoverageReport-$moduleName" + + JacocoReport task = project.tasks.register(taskName, JacocoReport).get() + task.description = "Generates JaCoCo coverage report for $moduleName" + + prepareJacocoReportTask(task, moduleName, rootTask) + + task.reports { + xml.required = true + xml.outputLocation = file("build/reports/jacoco/$moduleName/jacocoTestReport.xml") + html.required = true + html.outputLocation = file("build/reports/jacoco/$moduleName/html") + } + + return task +} + +/** + * Registers a JacocoCoverageVerification task based on the provided parameters. + * + * @param moduleName The module name to validate rules for. + * @param rootTask The root JacocoCoverageVerification task. + * @return The configured JacocoCoverageVerification task. + */ +private JacocoCoverageVerification registerJacocoTestCoverageVerification(String moduleName, JacocoCoverageVerification rootTask) { + def taskName = "jacocoTestCoverageVerification-$moduleName" + + def thresholds = ModuleCoverageThresholds[moduleName] + if (thresholds == null) { + println "No coverage thresholds defined for module '$moduleName'. Skipping verification..." + return null + } + def minInstructionCoveredRatio = thresholds["INSTRUCTION"] as double + def maxNumberUncoveredClasses = thresholds["CLASS"] as int + + JacocoCoverageVerification task = project.tasks.register(taskName, JacocoCoverageVerification).get() + task.description = "Validates JaCoCo coverage for vioalations for $moduleName" + + prepareJacocoReportTask(task, moduleName, rootTask) + applyVerificationRule(task, minInstructionCoveredRatio, maxNumberUncoveredClasses) + + return task +} + +/** + * Prepares a Jacoco report task (report & verification) to match a specific + * @param task that is modified + * @param moduleName of the module. + * @param rootTask the JacocoReportBase root task + */ +private void prepareJacocoReportTask(JacocoReportBase task, String moduleName, JacocoReportBase rootTask) { + task.group = "Reporting" + task.executionData = project.fileTree("${project.layout.buildDirectory.get()}/jacoco") { + include "test.exec" + } + + def modulePath = "$BasePath/$moduleName/**/*.class" + task.sourceDirectories.setFrom(project.files("src/main/java/$modulePath")) + task.classDirectories.setFrom( + files(rootTask.classDirectories.files.collect { classDir -> + project.fileTree(classDir) { + includes=[modulePath] + excludes=ignoredDirectories + } + }) + ) +} + +private static void applyVerificationRule(JacocoCoverageVerification task, double minInstructionCoveredRatio, int maxNumberUncoveredClasses) { + task.violationRules { + rule { + limit { + counter = "INSTRUCTION" + value = "COVEREDRATIO" + minimum = minInstructionCoveredRatio + } + limit { + counter = "CLASS" + value = "MISSEDCOUNT" + maximum = maxNumberUncoveredClasses + } + } + } +} diff --git a/supporting_scripts/generate_code_cov_table/.gitignore b/supporting_scripts/code-coverage/generate_code_cov_table/.gitignore similarity index 100% rename from supporting_scripts/generate_code_cov_table/.gitignore rename to supporting_scripts/code-coverage/generate_code_cov_table/.gitignore diff --git a/supporting_scripts/generate_code_cov_table/README.md b/supporting_scripts/code-coverage/generate_code_cov_table/README.md similarity index 100% rename from supporting_scripts/generate_code_cov_table/README.md rename to supporting_scripts/code-coverage/generate_code_cov_table/README.md diff --git a/supporting_scripts/generate_code_cov_table/env.example b/supporting_scripts/code-coverage/generate_code_cov_table/env.example similarity index 100% rename from supporting_scripts/generate_code_cov_table/env.example rename to supporting_scripts/code-coverage/generate_code_cov_table/env.example diff --git a/supporting_scripts/generate_code_cov_table/generate_code_cov_table.py b/supporting_scripts/code-coverage/generate_code_cov_table/generate_code_cov_table.py similarity index 100% rename from supporting_scripts/generate_code_cov_table/generate_code_cov_table.py rename to supporting_scripts/code-coverage/generate_code_cov_table/generate_code_cov_table.py diff --git a/supporting_scripts/generate_code_cov_table/requirements.txt b/supporting_scripts/code-coverage/generate_code_cov_table/requirements.txt similarity index 100% rename from supporting_scripts/generate_code_cov_table/requirements.txt rename to supporting_scripts/code-coverage/generate_code_cov_table/requirements.txt diff --git a/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py new file mode 100644 index 000000000000..50d21cf9c642 --- /dev/null +++ b/supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py @@ -0,0 +1,75 @@ +import os +import xml.etree.ElementTree as ET +import argparse + +def get_report_by_module(input_directory): + results = [] + for module_folder in os.listdir(input_directory): + module_path = os.path.join(input_directory, module_folder) + + if os.path.isdir(module_path): + report_file = os.path.join(module_path, f"jacocoTestReport.xml") + + if os.path.exists(report_file): + results.append({ + "module": module_folder, + "report_file": report_file + }) + else: + print(f"No XML report file found for module: {module_folder}. Skipping...") + + return results + + +def extract_coverage(reports): + results = [] + + for report in reports: + try: + tree = ET.parse(report['report_file']) + root = tree.getroot() + + instruction_counter = root.find("./counter[@type='INSTRUCTION']") + class_counter = root.find("./counter[@type='CLASS']") + + if instruction_counter == None or class_counter == None: + continue + + instruction_covered = int(instruction_counter.get('covered', 0)) + instruction_missed = int(instruction_counter.get('missed', 0)) + total_instructions = instruction_covered + instruction_missed + instruction_coverage = (instruction_covered / total_instructions * 100) if total_instructions > 0 else 0.0 + + missed_classes = int(class_counter.get('missed', 0)) + + results.append({ + "module": report['module'], + "instruction_coverage": instruction_coverage, + "missed_classes": missed_classes + }) + except Exception as e: + print(f"Error processing {report['module']}: {e}") + + results = sorted(results, key=lambda x: x['module']) + return results + + +def write_summary_to_file(coverage_by_module, output_file): + with open(output_file, "w") as f: + f.write("## Coverage Results\n\n") + f.write("| Module Name | Instruction Coverage (%) | Missed Classes |\n") + f.write("|-------------|---------------------------|----------------|\n") + for result in coverage_by_module: + f.write(f"| {result['module']} | {result['instruction_coverage']:.2f} | {result['missed_classes']} |\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process JaCoCo coverage reports.") + parser.add_argument("input_directory", type=str, help="Root directory containing JaCoCo coverage reports") + parser.add_argument("output_file", type=str, help="Output file to save the coverage results") + + args = parser.parse_args() + + reports = get_report_by_module(args.input_directory) + coverage_by_module = extract_coverage(reports) + write_summary_to_file(coverage_by_module, args.output_file) diff --git a/supporting_scripts/extract_number_of_server_starts.sh b/supporting_scripts/extract_number_of_server_starts.sh index 80f07e848207..a4e40eff07f2 100644 --- a/supporting_scripts/extract_number_of_server_starts.sh +++ b/supporting_scripts/extract_number_of_server_starts.sh @@ -9,8 +9,8 @@ then exit 1 fi -if [[ $numberOfStarts -ne 4 ]] +if [[ $numberOfStarts -gt 4 ]] then - echo "The number of Server Starts should be equal to 4! Please adapt this check if the change is intended or try to fix the underlying issue causing a different number of server starts!" + echo "The number of Server Starts should be lower than/equals 4! Please adapt this check if the change is intended or try to fix the underlying issue causing a different number of server starts!" exit 1 fi diff --git a/supporting_scripts/get_changed_modules.sh b/supporting_scripts/get_changed_modules.sh new file mode 100755 index 000000000000..d9188d69ee24 --- /dev/null +++ b/supporting_scripts/get_changed_modules.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Determines the changed modules following the module-directory structure for both main/test. +# Based on git-diff between the local state and an input branch name. +# Returns a comma-separated list of changed modules. +# Example: "./get_changed_modules.sh develop" + +# Check for the branch input argument. +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +BRANCH_TO_COMPARE="$1" + +MODULE_BASE_PATH="src/main/java/de/tum/cit/aet/artemis" +MODULE_BASE_TEST_PATH="src/test/java/de/tum/cit/aet/artemis" + +MODULES=$(find "$MODULE_BASE_PATH" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) +CHANGED_MODULES=() + +for MODULE in $MODULES; do + if git diff "$BRANCH_TO_COMPARE" --name-only | grep -q "^$MODULE_BASE_PATH/$MODULE/"; then + CHANGED_MODULES+=("$MODULE") + elif git diff "$BRANCH_TO_COMPARE" --name-only | grep -q "^$MODULE_BASE_TEST_PATH/$MODULE/"; then + CHANGED_MODULES+=("$MODULE") + fi +done + +IFS=, +echo "${CHANGED_MODULES[*]}"