Internal: Get handle to fuzz guidance in JQF test runner

Closes #82
rohanpadhye · Mar 22, 2021 · dc3919f · dc3919f
1 parent d41d87d
commit dc3919f
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 44 deletions.
diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/JQF.java
@@ -28,6 +28,8 @@
  */
 package edu.berkeley.cs.jqf.fuzz;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
@@ -36,7 +38,12 @@
 import com.pholser.junit.quickcheck.internal.generator.ServiceLoaderGeneratorSource;
 import com.pholser.junit.quickcheck.random.SourceOfRandomness;
 import com.pholser.junit.quickcheck.runner.JUnitQuickcheck;
+import edu.berkeley.cs.jqf.fuzz.guidance.Guidance;
+import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException;
+import edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing;
 import edu.berkeley.cs.jqf.fuzz.junit.quickcheck.FuzzStatement;
+import edu.berkeley.cs.jqf.fuzz.random.NoGuidance;
+import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance;
 import org.junit.runners.model.FrameworkMethod;
 import org.junit.runners.model.InitializationError;
 import org.junit.runners.model.Statement;
@@ -56,8 +63,7 @@ public JQF(Class<?> clazz) throws InitializationError {
         super(clazz);
         // Initialize generator repository with a deterministic seed (for reproducibility)
         SourceOfRandomness randomness = new SourceOfRandomness(new Random(42));
-        this.generatorRepository = new GeneratorRepository(randomness).register(new ServiceLoaderGeneratorSource());;
-
+        this.generatorRepository = new GeneratorRepository(randomness).register(new ServiceLoaderGeneratorSource());
     }
 
     @Override protected List<FrameworkMethod> computeTestMethods() {
@@ -83,9 +89,53 @@ private void validateFuzzMethods(List<Throwable> errors) {
     }
 
     @Override public Statement methodBlock(FrameworkMethod method) {
-        if (method.getAnnotation(Fuzz.class) != null) {
-            return new FuzzStatement(method, getTestClass(), generatorRepository);
+        // JQF only needs special handling for @Fuzz-annotated test methods
+        if (method.getAnnotation(Fuzz.class) == null) {
+            return super.methodBlock(method);
         }
-        return super.methodBlock(method);
+
+        // Get currently set fuzzing guidance
+        Guidance guidance = GuidedFuzzing.getCurrentGuidance();
+
+        // If nothing is set, default to random or repro
+        if (guidance == null) {
+            // Check for @Fuzz(repro=)
+            String repro = method.getAnnotation(Fuzz.class).repro();
+            if (repro.isEmpty()) {
+                guidance = new NoGuidance(GuidedFuzzing.DEFAULT_MAX_TRIALS, System.err);
+            } else {
+                String reproPath;
+                // Check if repro path is variable (e.g. `${foo}`)
+                if (repro.matches("\\$\\{[a-zA-Z.\\d_$]*\\}")) {
+                    // Get a system property with that name (e.g. `foo`)
+                    String key = repro.substring(2, repro.length()-1);
+                    String val = System.getProperty(key);
+
+                    // Check if such a property is set
+                    if (val == null) {
+                        throw new IllegalArgumentException(String.format("Test method has " +
+                                        "@Fuzz annotation with repro=%s, but such a system " +
+                                        "property is not set. Use `-D%s=<path>` when running.",
+                                repro, key));
+                    }
+
+                    reproPath = val;
+                } else {
+                    // If it is not a variable, then treat it literally
+                    reproPath = repro;
+                }
+
+                // Create a ReproGuidance with the given path
+                File inputFile = new File(reproPath);
+                try {
+                    guidance = new ReproGuidance(inputFile, null);
+                } catch (IOException e) {
+                    throw new GuidanceException(String.format("Could not open repro file: %s",
+                            inputFile.getAbsolutePath()), e);
+                }
+            }
+        }
+
+        return new FuzzStatement(method, getTestClass(), generatorRepository, guidance);
     }
 }
diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java
@@ -78,9 +78,10 @@ public class FuzzStatement extends Statement {
     private final GeneratorRepository generatorRepository;
     private final List<Class<?>> expectedExceptions;
     private final List<Throwable> failures = new ArrayList<>();
+    private final Guidance guidance;
 
     public FuzzStatement(FrameworkMethod method, TestClass testClass,
-                         GeneratorRepository generatorRepository) {
+                         GeneratorRepository generatorRepository, Guidance fuzzGuidance) {
         this.method = method;
         this.testClass = testClass;
         this.typeVariables =
@@ -89,7 +90,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass,
                         .genericsMap();
         this.generatorRepository = generatorRepository;
         this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes());
-
+        this.guidance = fuzzGuidance;
     }
 
 
@@ -106,43 +107,6 @@ public void evaluate() throws Throwable {
                 .map(this::produceGenerator)
                 .collect(Collectors.toList());
 
-        // Get the currently registered fuzz guidance
-        Guidance guidance = GuidedFuzzing.getCurrentGuidance();
-
-        // If nothing is set, default to random or repro
-        if (guidance == null) {
-            // Check for @Fuzz(repro=)
-            String repro = method.getAnnotation(Fuzz.class).repro();
-            if (repro.isEmpty()) {
-                guidance = new NoGuidance(GuidedFuzzing.DEFAULT_MAX_TRIALS, System.err);
-            } else {
-                String reproPath;
-                // Check if repro path is variable (e.g. `${foo}`)
-                if (repro.matches("\\$\\{[a-zA-Z.\\d_$]*\\}")) {
-                    // Get a system property with that name (e.g. `foo`)
-                    String key = repro.substring(2, repro.length()-1);
-                    String val = System.getProperty(key);
-
-                    // Check if such a property is set
-                    if (val == null) {
-                        throw new IllegalArgumentException(String.format("Test method has " +
-                                "@Fuzz annotation with repro=%s, but such a system " +
-                                "property is not set. Use `-D%s=<path>` when running.",
-                                repro, key));
-                    }
-
-                    reproPath = val;
-                } else {
-                    // If it is not a variable, then treat it literally
-                    reproPath = repro;
-                }
-
-                // Create a ReproGuidance with the given path
-                File inputFile = new File(reproPath);
-                guidance = new ReproGuidance(inputFile, null);
-            }
-        }
-
         // Keep fuzzing until no more input or I/O error with guidance
         try {