From d8636dbee44cd4aaaa48438532a8edcd7d9ce3c9 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Wed, 28 Aug 2024 14:12:22 -0400 Subject: [PATCH 01/14] Init type directed ChocoPy generator --- .../ChocoPySemanticGeneratorTypeDirected.java | 673 ++++++++++++++++++ .../chocopy/SemanticAnalysisTest.java | 6 +- 2 files changed, 677 insertions(+), 2 deletions(-) create mode 100644 examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java new file mode 100644 index 000000000..22dfc8bb6 --- /dev/null +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -0,0 +1,673 @@ +package edu.berkeley.cs.jqf.examples.chocopy; + +import com.pholser.junit.quickcheck.generator.GenerationStatus; +import com.pholser.junit.quickcheck.generator.Generator; +import com.pholser.junit.quickcheck.random.SourceOfRandomness; +import org.apache.commons.lang3.StringUtils; + + +import java.util.*; +import java.util.function.Function; +import org.apache.commons.lang3.tuple.Pair; +import chocopy.common.astnodes.ClassType; +import chocopy.common.analysis.types.*; + +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +/* Generates random strings that are syntactically valid ChocoPy */ +public class ChocoPySemanticGeneratorTypeDirected extends Generator { + public ChocoPySemanticGeneratorTypeDirected() { + super(String.class); // Register type of generated object + + // Read parameters from system properties + maxIdentifiers = Integer.getInteger(identifierProp, 3); + maxItems = Integer.getInteger(itemsProp, 3); + maxDepth = Integer.getInteger(depthProp, 3); + maxBound = 4; + assert(maxIdentifiers > 0); + assert(maxItems > 0); + assert(maxDepth > 0); + + // Create set of fixed identifiers + generateIdentifiers(maxBound); + } + + private final String identifierProp = "maxIdentifiers"; + private final String itemsProp = "maxItems"; + private final String depthProp = "maxDepth"; + private GenerationStatus status; // saved state object when generating + private static List identifiers; // Stores generated IDs, to promote re-use + private static List funcIdentifiers; // Stores generated IDs, to promote re-use + private static List allTypes; // Keeps track of all types + private static List classTypes; // Keeps track of all class types + private static Map varTypes; // Keeps track of variables and their types + private static Map funcTypes; // Keeps track of functions and their return types + private static int maxIdentifiers; + private static int maxItems; + private static int maxDepth; + private static int maxBound; + private int identCounter; + private int funcIdentCounter; + private int statementDepth; // Keeps track of how deep the AST is at any point + private int declarationDepth; // Keeps track of how deep the AST is at any point + private int expressionDepth; // Keeps track of how nested an expression is at any point + private int indentLevel; // Keeps track of indentation level + + private static final String[] INT_BINARY_OP_TOKENS = { + "+", "-", "*", "//", "%", + }; + + private static final String[] BOOL_BINARY_OP_TOKENS = { + "and", "or", "==", "!=", "is", + "<", "<=", ">", ">=", "==", "!=", "is" + }; + + private static final String[] BINARY_BOOL_TOKENS = { + "and", "or" + }; + + private static final String[] INT_LITERALS = { + "0", "1" + }; + + private static final String[] STRING_LITERALS = { + "\"a\"", "\"\"" + }; + + private static final String[] BOOL_LITERALS = { + "True", "False" + }; + + private static final Type[] PRIMITIVE_TYPES = { + Type.INT_TYPE, + Type.STR_TYPE, + Type.BOOL_TYPE, + }; + + private static final ValueType[] BASE_TYPES = { + Type.INT_TYPE, + Type.STR_TYPE, + Type.BOOL_TYPE, + Type.OBJECT_TYPE, + + }; + + private static final String INDENT_TOKEN = " "; // 4 spaces + + + /** Main entry point. Called once per test case. Returns a random ChocoPy program. */ + @Override + public String generate(SourceOfRandomness random, GenerationStatus status) { + this.status = status; // we save this so that we can pass it on to other generators + this.declarationDepth = 0; + this.statementDepth = 0; + this.expressionDepth = 0; + this.indentLevel = 0; + this.classTypes = new ArrayList<>(); + this.allTypes = Arrays.asList(BASE_TYPES); + this.varTypes = new HashMap<>(); + this.funcTypes = new HashMap<>(); + this.identCounter = 0; + this.funcIdentCounter = 0; + return generateProgram(random); + } + + /** Utility method for generating a random list of items (e.g. statements, arguments, attributes) */ + private static List generateItems(Function genMethod, SourceOfRandomness random, int minimum) { + int len = nextIntBound(random, minimum, maxBound, maxItems); + List items = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + items.add(genMethod.apply(random)); + } + return items; + } + + /** Utility method for generating a random list of items from a list of functions to choose from */ + private static List generateItemsMultipleMethods(List> genMethods, SourceOfRandomness random, int minimum) { + int len = nextIntBound(random, minimum, maxBound, maxItems); + List items = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + items.add(random.choose(genMethods).apply(random)); + } + return items; + } + + private static int nextIntBound(SourceOfRandomness random, int minimum, int maximum, int maxParam) { + int randInt = random.nextInt(minimum, maximum); + assumeTrue(randInt <= maxParam); + return randInt; + } + + /** Generates a random ChocoPy program of classes, declarations, and statements */ + private String generateProgram(SourceOfRandomness random) { + String declarations = String.join("", generateItemsMultipleMethods(Arrays.asList( +// this::generateClassDef, + this::generateFuncDef, + this::generateVarDef + ), random, 0)); + String statements = generateBlock(random, 0); + return declarations + statements; + } + + /** Generates a random ChocoPy declaration */ + private String generateDeclaration(SourceOfRandomness random) { + String result = StringUtils.repeat(INDENT_TOKEN, indentLevel); + int randDepth = nextIntBound(random, 0, maxBound, maxDepth); + if (declarationDepth >= randDepth) { + // Choose a random private method from this class, and then call it with `random` + result += generateVarDef(random); + } else { + // If depth is low and we won the flip, then generate compound declarations + // (that is, declarations that contain other declarations) + result += random.choose(Arrays.>asList( + this::generateVarDef +// this::generateFuncDef + )).apply(random); + } + return result; + } + + /** Generates a random ChocoPy function declaration */ + private String generateFuncDeclaration(SourceOfRandomness random) { + String result = StringUtils.repeat(INDENT_TOKEN, indentLevel); + int randDepth = nextIntBound(random, 0, maxBound, maxDepth); + if (declarationDepth >= randDepth) { + // Choose a random private method from this class, and then call it with `random` + result += random.choose(Arrays.>asList( + this::generateVarDef +// this::generateNonlocalDecl, +// this::generateGlobalDecl + )).apply(random); + } else { + // If depth is low and we won the flip, then generate compound declarations + // (that is, declarations that contain other declarations) + result += random.choose(Arrays.>asList( + this::generateVarDef, + this::generateFuncDef + )).apply(random); + } + return result + "\n"; + } + + /** Generates a random ChocoPy statement */ + private String generateStatement(SourceOfRandomness random) { + String result = StringUtils.repeat(INDENT_TOKEN, indentLevel); + // If depth is too high, then generate only simple statements to prevent infinite recursion + // If not, generate simple statements after the flip of a coin + int randDepth = nextIntBound(random, 0, maxBound, maxDepth); + if (statementDepth >= randDepth) { + // Choose a random private method from this class, and then call it with `random` + result += random.choose(Arrays.>asList( + this::generateAssignStmt, + this::generatePassStmt, + this::generateExpressionStmt + )).apply(random); + } else { + // If depth is low and we won the flip, then generate compound statements + // (that is, statements that contain other statements) + result += random.choose(Arrays.>asList( + this::generateIfStmt, + this::generateForStmt, + this::generateWhileStmt + )).apply(random); + } + return result + "\n"; + } + + private String generateVarOfType(SourceOfRandomness random, Type type) { + List candidateVars = new ArrayList<>(); + for (Map.Entry entry : varTypes.entrySet()) { + if (entry.getValue().equals(type)) { + candidateVars.add(entry.getKey()); + } + } + assumeTrue(!candidateVars.isEmpty()); + return random.choose(candidateVars); + } + + private String generateParenExprOfType(SourceOfRandomness random, Type type) { + return "(" + generateExpressionOfType(random, type) + ")"; + } + + // TODO: implement + private String generateMemberExprOfType(SourceOfRandomness random, Type type) { + return ""; + } + + // TODO: implement + private String generateMethodCallExprOfType(SourceOfRandomness random, Type type) { + return ""; + } + + private String generateIndexExprOfType(SourceOfRandomness random, Type type) { + System.out.println("Generating index expr of type: " + type); + Type listType = new ListValueType(type); + System.out.println("Generating list expr of type: " + listType); + String result = ""; + result = generateListExprOfType(random, listType); + String index = generateExpressionOfType(random, Type.INT_TYPE); + result += "[" + index + "]"; + return result; + } + + private String generateCallExprOfType(SourceOfRandomness random, Type type) { + // Need to find a function that returns the correct type + List candidateFuncs = new ArrayList<>(); + for (Map.Entry entry : funcTypes.entrySet()) { + Type returnType = entry.getValue().returnType; + if (returnType.equals(type)) { + candidateFuncs.add(entry.getKey()); + } + } + assumeTrue(!candidateFuncs.isEmpty()); + int funcIndex = random.nextInt(0, candidateFuncs.size()); + String func = candidateFuncs.get(funcIndex); + FuncType funcType = funcTypes.get(func); + List paramTypes = funcTypes.get(func).parameters; + List args = new ArrayList<>(); + for (Type paramType : paramTypes) { + args.add(generateExpressionOfType(random, paramType)); + } + return func + "(" + String.join(", ", args) + ")"; + } + + private String generateBinaryExprOfType(SourceOfRandomness random, Type type) { + System.out.println("Generating binary expr of type: " + type); + Type operandType = null; + String token = ""; + if (type == Type.INT_TYPE) { + token = random.choose(INT_BINARY_OP_TOKENS); + operandType = Type.INT_TYPE; + } else if (type == Type.BOOL_TYPE) { + token = random.choose(BOOL_BINARY_OP_TOKENS); + switch(token) { + case "and": + case "or": + operandType = Type.BOOL_TYPE; + break; + case "-": + case "<": + case "<=": + case ">": + case ">=": + case "*": + case "//": + case "%": + operandType = Type.INT_TYPE; + break; + case "==": + case "!=": + operandType = random.choose(PRIMITIVE_TYPES); + break; + case "is": + List nonPrimitiveTypes = new ArrayList<>(classTypes); + nonPrimitiveTypes.add(Type.OBJECT_TYPE); + nonPrimitiveTypes.add(new ListValueType(Type.INT_TYPE)); + operandType = random.choose(nonPrimitiveTypes); + if (operandType.isListType()) { + int listDepth = random.nextInt(0, maxDepth); + operandType = random.choose(allTypes); + for (int i = 0; i < listDepth; i++) { + operandType = new ListValueType(operandType); + } + } + break; + default: + System.out.println("Invalid token: " + token); + System.out.println("Invalid type: " + type); + assert(false); + } + } else if (type == Type.STR_TYPE) { + token = "+"; + operandType = Type.STR_TYPE; + } else if (type.isListType()) { + token = "+"; + operandType = type; + } else { + assert(false); + } + String lhs = generateExpressionOfType(random, operandType); + String rhs = generateExpressionOfType(random, operandType); + return lhs + " " + token + " " + rhs; + } + + private String generateUnaryExprOfType(SourceOfRandomness random, Type type) { + Type operandType = null; + String token = ""; + if (type == Type.INT_TYPE) { + token = "-"; + operandType = Type.INT_TYPE; + } else if (type == Type.BOOL_TYPE) { + token = "not"; + operandType = Type.BOOL_TYPE; + } else { + assert(false); + } + return token + " " + generateExpressionOfType(random, operandType); + } + + private String generateListExprOfType(SourceOfRandomness random, Type type) { + assert(type.isListType()); + String result = random.choose(Arrays.>asList( + r -> generateConstExprOfType(r, type), + r -> generateVarOfType(r, type), + r -> generateParenExprOfType(r, type), +// r -> generateMemberExprOfType(r, type), +// r -> generateMethodCallExprOfType(r, type), + r -> generateIndexExprOfType(r, type), + r -> generateCallExprOfType(r, type), + r -> generateBinaryExprOfType(r, type) + )).apply(random); + return result; + } + + + /** Generates a random ChocoPy expression using recursive calls */ + private String generateExpressionOfType(SourceOfRandomness random, Type type) { + String result; + // Choose terminal if nesting depth is too high or based on a random flip of a coin + int randDepth = nextIntBound(random, 0, maxBound, maxDepth); + if (expressionDepth >= randDepth) { + result = generateConstExprOfType(random, type); + } else { + expressionDepth++; + // Otherwise, choose a non-terminal generating function + if (type == Type.INT_TYPE || type == Type.BOOL_TYPE) { + result = random.choose(Arrays.>asList( + r -> generateConstExprOfType(r, type), + r -> generateVarOfType(r, type), + r -> generateParenExprOfType(r, type), +// r -> generateMemberExprOfType(r, type), +// r -> generateMethodCallExprOfType(r, type), + r -> generateIndexExprOfType(r, type), + r -> generateCallExprOfType(r, type), + r -> generateBinaryExprOfType(r, type), + r -> generateUnaryExprOfType(r, type) + )).apply(random); + } else if (type.isListType() || type == Type.STR_TYPE) { + result = random.choose(Arrays.>asList( + r -> generateConstExprOfType(r, type), + r -> generateVarOfType(r, type), + r -> generateParenExprOfType(r, type), +// r -> generateMemberExprOfType(r, type), +// r -> generateMethodCallExprOfType(r, type), + r -> generateIndexExprOfType(r, type), + r -> generateCallExprOfType(r, type), + r -> generateBinaryExprOfType(r, type) + )).apply(random); + } else { + result = random.choose(Arrays.>asList( + r -> generateConstExprOfType(r, type), + r -> generateVarOfType(r, type), + r -> generateParenExprOfType(r, type), +// r -> generateMemberExprOfType(r, type), +// r -> generateMethodCallExprOfType(r, type), + r -> generateIndexExprOfType(r, type), + r -> generateCallExprOfType(r, type) + )).apply(random); + } + expressionDepth--; + } + return result; + } + + // TODO: Add list type + private String generateConstExprOfType(SourceOfRandomness random, Type type) { + if (type == Type.INT_TYPE) { + return generateIntLiteral(random); + } else if (type == Type.STR_TYPE) { + return generateStringLiteral(random); + } else if (type == Type.BOOL_TYPE) { + return generateBoolLiteral(random); + } else if (type.isListType()) { + return "[" + generateConstExprOfType(random, type.elementType()) + "]"; + } else { + assumeTrue(false); + return ""; + } + } + + // TODO: Add member expressions and method calls + private String generateAssignStmt(SourceOfRandomness random) { + String result = ""; + // Choose a random entry in varTypes + assumeTrue(!varTypes.isEmpty()); + String var = random.choose(varTypes.entrySet()).getKey(); + Type varType = varTypes.get(var); + result += var + " = "; + + return result + generateExpressionOfType(random, varType); + } + + /** Generates a block of statements, excluding the statement */ + private String generateBlock(SourceOfRandomness random, int minimum) { + return String.join("", generateItems(this::generateStatement, random, minimum)); + } + +// private String generateClassDef(SourceOfRandomness random) { +// String result = ""; +// String className = generateIdentifier(random); +// // Superclass could be one of the identifiers or object. Index should be from 0 to maxIdentifiers inclusive. +// int superClassIndex = nextIntBound(random, 0, classTypes.size(), maxIdentifiers); +// String superClassName = classTypes.get(superClassIndex); +// result += "class " + className + "(" + superClassName + "):\n"; +// indentLevel++; +// result += generateDeclarationBlock(random, 1); +// indentLevel--; +// return result + "\n"; +// } + + /** Generates a block of VarDefs and FuncDefs*/ + private String generateDeclarationBlock(SourceOfRandomness random, int minimum) { + return String.join("", generateItems(this::generateDeclaration, random, minimum)); + } + + private String generateExpressionStmt(SourceOfRandomness random) { + Type randomType = random.choose(allTypes); + return generateExpressionOfType(random, randomType); + } + + private String generateForStmt(SourceOfRandomness random) { + statementDepth++; + ValueType listType = generateListType(random); + String var = generateVarOfType(random, listType.elementType()); + String s = "for " + var + " in " + generateExpressionOfType(random, listType) + ":\n"; + indentLevel++; + s += generateBlock(random, 1); + indentLevel--; + statementDepth--; + return s; + } + + private String generateFuncDef(SourceOfRandomness random) { + declarationDepth++; + + String funcIdent = generateFuncIdentifier(random); + int numParams = nextIntBound(random, 0, maxItems, maxItems); + StringBuilder result = new StringBuilder("def " + generateFuncIdentifier(random) + "("); + List paramTypes = new ArrayList<>(); + for (int i = 0; i < numParams; i++) { + Pair param = generateTypedVar(random); + paramTypes.add(param.getRight()); + result.append(param.getLeft()).append(":").append(param.getRight()); + } + + ValueType returnType = Type.NONE_TYPE; + + if (random.nextBoolean()) { + returnType = generateType(random); + result.append("->").append(returnType); + } + FuncType funcType = new FuncType(paramTypes, returnType); + result.append(")"); + + + funcTypes.put(funcIdent, funcType); + + result.append(":\n"); + indentLevel++; + result.append(String.join("", generateItems(this::generateFuncDeclaration, random, 0))); + result.append(generateBlock(random, 1)); + if (returnType != Type.NONE_TYPE) { + result.append(generateReturnStmtOfType(random, returnType)); + } + indentLevel--; + declarationDepth--; + return result + "\n"; + } + + private String generateGlobalDecl(SourceOfRandomness random) { + return "global " + generateIdentifier(random) + "\n"; + } + + private String generateIdentifier(SourceOfRandomness random) { + return "a" + identCounter++; + } + + private String generateFuncIdentifier(SourceOfRandomness random) { + return "b" + funcIdentCounter++; + } + + /** Creates initial set of identifiers depending on parameter */ + private void generateIdentifiers(int numIdentifiers) { + this.identifiers = new ArrayList<>(); + this.funcIdentifiers = new ArrayList<>(); + funcIdentifiers.add("len"); + funcIdentifiers.add("print"); + String ident; + for (int i = 0; i < numIdentifiers; i++) { + ident = "a" + i; + identifiers.add(ident); + funcIdentifiers.add(ident); + } + } + + private String generateIfExprOfType(SourceOfRandomness random, Type type) { + return generateExpressionOfType(random, type) + " if " + generateExpressionOfType(random, Type.BOOL_TYPE) + " else " + generateExpressionOfType(random, type); + } + + private String generateIfStmt(SourceOfRandomness random) { + statementDepth++; + String result = "if " + generateExpressionOfType(random, Type.BOOL_TYPE) + ":\n"; + indentLevel++; + result += generateBlock(random, 1); + indentLevel--; + if (random.nextBoolean()) { + result += StringUtils.repeat(INDENT_TOKEN, indentLevel); + result += "elif " + generateExpressionOfType(random, Type.BOOL_TYPE) + ":\n"; + indentLevel++; + result += generateBlock(random, 1); + indentLevel--; + } + if (random.nextBoolean()) { + result += StringUtils.repeat(INDENT_TOKEN, indentLevel); + result += "else:\n"; + indentLevel++; + result += generateBlock(random, 1); + indentLevel--; + } + statementDepth--; + return result; + } + + // Generate fixed primitive literals + private String generateLiteralOfType(SourceOfRandomness random, Type type) { + + switch (type.toString()) { + case "int": + return generateIntLiteral(random); + case "str": + return generateStringLiteral(random); + case "bool": + return generateBoolLiteral(random); + default: + return generateNoneLiteral(random); + } + } + + private String generateIntLiteral(SourceOfRandomness random) { + return random.choose(INT_LITERALS); + } + + private String generateStringLiteral(SourceOfRandomness random) { + return random.choose(STRING_LITERALS); + } + + private String generateBoolLiteral(SourceOfRandomness random) { + return random.choose(BOOL_LITERALS); + } + + private String generateNoneLiteral(SourceOfRandomness random) { + return "None"; + } + +// private String generateMemberExpr(SourceOfRandomness random) { +// return "(" + generateCExpression(random) + ")." + generateIdentifier(random); +// } +// +// private String generateMethodCallExpr(SourceOfRandomness random) { +// return generateCExpression(random) + "." + generateCallExpr(random); +// } +// +// private String generateNonlocalDecl(SourceOfRandomness random) { +// return "nonlocal " + generateIdentifier(random) + "\n"; +// } +// + + private String generatePassStmt(SourceOfRandomness random) { + return "pass"; + } + + private String generateReturnStmtOfType(SourceOfRandomness random, Type type) { + return "return " + generateExpressionOfType(random, type); + } + + /** Randomly choose from types and random list depth using maxDepth parameter */ + private ValueType generateType(SourceOfRandomness random) { + ValueType baseType = random.choose(allTypes); + int listDepth = random.nextInt(0, maxDepth); + if (listDepth == 0) { + return baseType; + } else { + for (int i = 0; i < listDepth; i++) { + baseType = new ListValueType(baseType); + } + } + return baseType; + } + + /** Randomly choose from types and random list depth using maxDepth parameter */ + private ValueType generateListType(SourceOfRandomness random) { + ValueType baseType = random.choose(allTypes); + int listDepth = random.nextInt(1, maxDepth); + for (int i = 0; i < listDepth; i++) { + baseType = new ListValueType(baseType); + } + return baseType; + } + + private Pair generateTypedVar(SourceOfRandomness random) { + ValueType type = generateType(random); + String ident = generateIdentifier(random); + varTypes.put(ident, type); + return Pair.of(ident, type); + } + + private String generateVarDef(SourceOfRandomness random) { + Pair typedVar = generateTypedVar(random); + String ident = typedVar.getLeft(); + ValueType varType = typedVar.getRight(); + + return ident + ":" + varType + " = " + generateLiteralOfType(random, varType) + "\n"; + } + + private String generateWhileStmt(SourceOfRandomness random) { + statementDepth++; + indentLevel++; + String result = "while " + generateExpressionOfType(random, Type.BOOL_TYPE) + ":\n" + generateBlock(random, 1); + indentLevel--; + statementDepth--; + return result; + } +} diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java index 2fb3b44b2..090ec6ef0 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java @@ -9,6 +9,7 @@ import edu.berkeley.cs.jqf.fuzz.JQF; import org.junit.runner.RunWith; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -17,9 +18,10 @@ public class SemanticAnalysisTest { /** Entry point for fuzzing reference ChocoPy semantic analysis with ChocoPy code generator */ @Fuzz - public void fuzzSemanticAnalysis(@From(ChocoPySemanticGenerator.class) String code) { + public void fuzzSemanticAnalysis(@From(ChocoPySemanticGeneratorTypeDirected.class) String code) { Program program = RefParser.process(code, false); assumeTrue(!program.hasErrors()); - RefAnalysis.process(program); + Program typedProgram = RefAnalysis.process(program); + assertFalse(typedProgram.hasErrors()); } } From 890719a82723c44bdfccee199e33481f01b1fa8f Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Fri, 30 Aug 2024 13:04:31 -0400 Subject: [PATCH 02/14] Initial Tyche logging support --- fuzz/pom.xml | 6 +++ .../fuzz/junit/quickcheck/FuzzStatement.java | 51 +++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/fuzz/pom.xml b/fuzz/pom.xml index 29ee1a002..d7cbf3c43 100644 --- a/fuzz/pom.xml +++ b/fuzz/pom.xml @@ -55,6 +55,12 @@ eclipse-collections 10.4.0 + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 + compile + diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 4ba89fd53..8162bb368 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -30,24 +30,37 @@ package edu.berkeley.cs.jqf.fuzz.junit.quickcheck; import java.io.EOFException; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.lang.reflect.Parameter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.internal.ParameterTypeContext; import com.pholser.junit.quickcheck.internal.generator.GeneratorRepository; import com.pholser.junit.quickcheck.random.SourceOfRandomness; +import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance; import edu.berkeley.cs.jqf.fuzz.guidance.Guidance; import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException; import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException; import edu.berkeley.cs.jqf.fuzz.guidance.Result; import edu.berkeley.cs.jqf.fuzz.guidance.StreamBackedRandom; import edu.berkeley.cs.jqf.fuzz.junit.TrialRunner; +import edu.berkeley.cs.jqf.fuzz.random.NoGuidance; +import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance; import edu.berkeley.cs.jqf.instrument.InstrumentationException; +import edu.berkeley.cs.jqf.fuzz.util.Observability; import org.junit.AssumptionViolatedException; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.MultipleFailureException; @@ -73,6 +86,7 @@ public class FuzzStatement extends Statement { private final List> expectedExceptions; private final List failures = new ArrayList<>(); private final Guidance guidance; + private final Observability observability; private boolean skipExceptionSwallow; public FuzzStatement(FrameworkMethod method, TestClass testClass, @@ -85,6 +99,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass, this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes()); this.guidance = fuzzGuidance; this.skipExceptionSwallow = Boolean.getBoolean("jqf.failOnDeclaredExceptions"); + this.observability = new Observability(testClass.getName(), method.getName(), System.currentTimeMillis()); } /** @@ -101,16 +116,19 @@ public void evaluate() throws Throwable { .collect(Collectors.toList()); // Keep fuzzing until no more input or I/O error with guidance + // Get current time in unix timestamp + long endGenerationTime = 0; try { // Keep fuzzing as long as guidance wants to while (guidance.hasInput()) { Result result = INVALID; Throwable error = null; + long startTrialTime = System.currentTimeMillis(); // Initialize guided fuzzing using a file-backed random number source + Object [] args = null; try { - Object[] args; try { // Generate input values @@ -142,6 +160,7 @@ public void evaluate() throws Throwable { throw new GuidanceException(e); } + endGenerationTime = System.currentTimeMillis(); // Attempt to run the trial guidance.run(testClass, method, args); @@ -170,6 +189,33 @@ public void evaluate() throws Throwable { failures.add(e); } } + long endTrialTime = System.currentTimeMillis(); + if (System.getProperty("jqfObservability") != null) { + + + // - "status": "passed", "failed", or "gave_up" + observability.addStatus(result); + if (result == SUCCESS) { + observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); + } + observability.add("representation", Arrays.toString(args)); + // - "status_reason": If non-empty, the reason for which the test failed or was abandoned. + + // - "how_generated": "Zest", "blind", or "repro" + if (guidance instanceof ZestGuidance) { + observability.add("how_generated", "Zest"); + } else if (guidance instanceof NoGuidance) { + observability.add("how_generated", "random"); + } else if (guidance instanceof ReproGuidance) { + observability.add("how_generated", "repro"); + } else { + observability.add("how_generated", "unknown"); + } + + observability.writeToFile(); + + + } // Inform guidance about the outcome of this trial try { @@ -180,8 +226,6 @@ public void evaluate() throws Throwable { // Anything else thrown from handleResult is an internal error, so wrap throw new GuidanceException(e); } - - } } catch (GuidanceException e) { System.err.println("Fuzzing stopped due to guidance exception: " + e.getMessage()); @@ -198,6 +242,7 @@ public void evaluate() throws Throwable { } } + } /** From 3f324fac35195463ea3c4a67d199d628af532458 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Fri, 30 Aug 2024 13:04:31 -0400 Subject: [PATCH 03/14] Initial Tyche logging support --- fuzz/pom.xml | 6 +++ .../fuzz/junit/quickcheck/FuzzStatement.java | 51 +++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/fuzz/pom.xml b/fuzz/pom.xml index 29ee1a002..d7cbf3c43 100644 --- a/fuzz/pom.xml +++ b/fuzz/pom.xml @@ -55,6 +55,12 @@ eclipse-collections 10.4.0 + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 + compile + diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 4ba89fd53..8162bb368 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -30,24 +30,37 @@ package edu.berkeley.cs.jqf.fuzz.junit.quickcheck; import java.io.EOFException; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.lang.reflect.Parameter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.internal.ParameterTypeContext; import com.pholser.junit.quickcheck.internal.generator.GeneratorRepository; import com.pholser.junit.quickcheck.random.SourceOfRandomness; +import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance; import edu.berkeley.cs.jqf.fuzz.guidance.Guidance; import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException; import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException; import edu.berkeley.cs.jqf.fuzz.guidance.Result; import edu.berkeley.cs.jqf.fuzz.guidance.StreamBackedRandom; import edu.berkeley.cs.jqf.fuzz.junit.TrialRunner; +import edu.berkeley.cs.jqf.fuzz.random.NoGuidance; +import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance; import edu.berkeley.cs.jqf.instrument.InstrumentationException; +import edu.berkeley.cs.jqf.fuzz.util.Observability; import org.junit.AssumptionViolatedException; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.MultipleFailureException; @@ -73,6 +86,7 @@ public class FuzzStatement extends Statement { private final List> expectedExceptions; private final List failures = new ArrayList<>(); private final Guidance guidance; + private final Observability observability; private boolean skipExceptionSwallow; public FuzzStatement(FrameworkMethod method, TestClass testClass, @@ -85,6 +99,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass, this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes()); this.guidance = fuzzGuidance; this.skipExceptionSwallow = Boolean.getBoolean("jqf.failOnDeclaredExceptions"); + this.observability = new Observability(testClass.getName(), method.getName(), System.currentTimeMillis()); } /** @@ -101,16 +116,19 @@ public void evaluate() throws Throwable { .collect(Collectors.toList()); // Keep fuzzing until no more input or I/O error with guidance + // Get current time in unix timestamp + long endGenerationTime = 0; try { // Keep fuzzing as long as guidance wants to while (guidance.hasInput()) { Result result = INVALID; Throwable error = null; + long startTrialTime = System.currentTimeMillis(); // Initialize guided fuzzing using a file-backed random number source + Object [] args = null; try { - Object[] args; try { // Generate input values @@ -142,6 +160,7 @@ public void evaluate() throws Throwable { throw new GuidanceException(e); } + endGenerationTime = System.currentTimeMillis(); // Attempt to run the trial guidance.run(testClass, method, args); @@ -170,6 +189,33 @@ public void evaluate() throws Throwable { failures.add(e); } } + long endTrialTime = System.currentTimeMillis(); + if (System.getProperty("jqfObservability") != null) { + + + // - "status": "passed", "failed", or "gave_up" + observability.addStatus(result); + if (result == SUCCESS) { + observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); + } + observability.add("representation", Arrays.toString(args)); + // - "status_reason": If non-empty, the reason for which the test failed or was abandoned. + + // - "how_generated": "Zest", "blind", or "repro" + if (guidance instanceof ZestGuidance) { + observability.add("how_generated", "Zest"); + } else if (guidance instanceof NoGuidance) { + observability.add("how_generated", "random"); + } else if (guidance instanceof ReproGuidance) { + observability.add("how_generated", "repro"); + } else { + observability.add("how_generated", "unknown"); + } + + observability.writeToFile(); + + + } // Inform guidance about the outcome of this trial try { @@ -180,8 +226,6 @@ public void evaluate() throws Throwable { // Anything else thrown from handleResult is an internal error, so wrap throw new GuidanceException(e); } - - } } catch (GuidanceException e) { System.err.println("Fuzzing stopped due to guidance exception: " + e.getMessage()); @@ -198,6 +242,7 @@ public void evaluate() throws Throwable { } } + } /** From 0ccfcd1003b876af20eabbc048d6bfaf3372c20a Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Fri, 30 Aug 2024 13:15:08 -0400 Subject: [PATCH 04/14] Cleanup --- .../fuzz/junit/quickcheck/FuzzStatement.java | 18 +--- .../cs/jqf/fuzz/util/Observability.java | 100 ++++++++++++++++++ 2 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 8162bb368..45d15baef 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -30,21 +30,12 @@ package edu.berkeley.cs.jqf.fuzz.junit.quickcheck; import java.io.EOFException; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import java.lang.reflect.Parameter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.internal.ParameterTypeContext; @@ -127,7 +118,7 @@ public void evaluate() throws Throwable { long startTrialTime = System.currentTimeMillis(); // Initialize guided fuzzing using a file-backed random number source - Object [] args = null; + Object [] args = {}; try { try { @@ -191,17 +182,12 @@ public void evaluate() throws Throwable { } long endTrialTime = System.currentTimeMillis(); if (System.getProperty("jqfObservability") != null) { - - - // - "status": "passed", "failed", or "gave_up" observability.addStatus(result); if (result == SUCCESS) { observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); } observability.add("representation", Arrays.toString(args)); - // - "status_reason": If non-empty, the reason for which the test failed or was abandoned. - // - "how_generated": "Zest", "blind", or "repro" if (guidance instanceof ZestGuidance) { observability.add("how_generated", "Zest"); } else if (guidance instanceof NoGuidance) { @@ -213,8 +199,6 @@ public void evaluate() throws Throwable { } observability.writeToFile(); - - } // Inform guidance about the outcome of this trial diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java new file mode 100644 index 000000000..a8977948a --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java @@ -0,0 +1,100 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.berkeley.cs.jqf.fuzz.guidance.Result; + +import java.io.FileWriter; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE; +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID; + +public class Observability { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final String testClass; + private final String testMethod; + private final Path obsPath; + private static ObjectNode testCaseJsonObject; + private final long startTime; + + public Observability(String testClass, String testMethod, long startTime) { + this.testClass = testClass; + this.testMethod = testMethod; + this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl"); + if (obsPath.toFile().exists()) { + obsPath.toFile().delete(); + } + this.startTime = startTime; + this.initializeTestCase(); + } + + public void initializeTestCase() { + testCaseJsonObject = objectMapper.createObjectNode(); + testCaseJsonObject.putObject("features"); + testCaseJsonObject.putObject("timing"); + testCaseJsonObject.putObject("coverage"); + testCaseJsonObject.putObject("args"); + testCaseJsonObject.putObject("metadata"); + testCaseJsonObject.put("type", "test_case"); + testCaseJsonObject.put("run_start", startTime); + testCaseJsonObject.put("property", testMethod); + } + + public static void event(String value, Object payload) throws RuntimeException { + // Add the payload to the features object + JsonNode jsonFeaturesNode = testCaseJsonObject.get("features"); + ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode; + + if (payload instanceof Integer) { + featuresNode.put(value, (Integer) payload); + } else if (payload instanceof String) { + featuresNode.put(value, (String) payload); + } else if (payload instanceof Float) { + featuresNode.put(value, (Float) payload); + } else { + throw new RuntimeException("Unsupported payload type for event"); + } + } + + public void addStatus(Result result) { + if (result == INVALID) { + testCaseJsonObject.put("status", "gave_up"); + testCaseJsonObject.put("status_reason", "assumption violated"); + } else if (result == FAILURE) { + testCaseJsonObject.put("status", "failed"); + testCaseJsonObject.put("status_reason", "Encountered exception"); + } else { + testCaseJsonObject.put("status", "passed"); + testCaseJsonObject.put("status_reason", ""); + } + + } + + public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) { + JsonNode timingNode = testCaseJsonObject.get("timing"); + ObjectNode timingObject = (ObjectNode) timingNode; + timingObject.put("generation", endGenerationTime - startTime); + timingObject.put("execution", endExecutionTime - endGenerationTime); + } + + public void add(String key, String value) { + testCaseJsonObject.put(key, value); + } + + public void writeToFile() { + // Append the JSON object to a file followed by a newline + try { + String jsonString = objectMapper.writeValueAsString(testCaseJsonObject); + try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) { + writer.write(jsonString); + writer.write(System.lineSeparator()); // Add a new line after each object + } + } catch (Exception e) { + throw new RuntimeException("Failed to write observations to file", e); + } + } +} From 8ba7e1a646b9523829c68ba5f1e4b8b7b4e7938d Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Fri, 30 Aug 2024 13:15:08 -0400 Subject: [PATCH 05/14] Cleanup --- .../fuzz/junit/quickcheck/FuzzStatement.java | 18 +--- .../cs/jqf/fuzz/util/Observability.java | 100 ++++++++++++++++++ 2 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 8162bb368..45d15baef 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -30,21 +30,12 @@ package edu.berkeley.cs.jqf.fuzz.junit.quickcheck; import java.io.EOFException; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import java.lang.reflect.Parameter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.internal.ParameterTypeContext; @@ -127,7 +118,7 @@ public void evaluate() throws Throwable { long startTrialTime = System.currentTimeMillis(); // Initialize guided fuzzing using a file-backed random number source - Object [] args = null; + Object [] args = {}; try { try { @@ -191,17 +182,12 @@ public void evaluate() throws Throwable { } long endTrialTime = System.currentTimeMillis(); if (System.getProperty("jqfObservability") != null) { - - - // - "status": "passed", "failed", or "gave_up" observability.addStatus(result); if (result == SUCCESS) { observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); } observability.add("representation", Arrays.toString(args)); - // - "status_reason": If non-empty, the reason for which the test failed or was abandoned. - // - "how_generated": "Zest", "blind", or "repro" if (guidance instanceof ZestGuidance) { observability.add("how_generated", "Zest"); } else if (guidance instanceof NoGuidance) { @@ -213,8 +199,6 @@ public void evaluate() throws Throwable { } observability.writeToFile(); - - } // Inform guidance about the outcome of this trial diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java new file mode 100644 index 000000000..a8977948a --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java @@ -0,0 +1,100 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.berkeley.cs.jqf.fuzz.guidance.Result; + +import java.io.FileWriter; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE; +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID; + +public class Observability { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final String testClass; + private final String testMethod; + private final Path obsPath; + private static ObjectNode testCaseJsonObject; + private final long startTime; + + public Observability(String testClass, String testMethod, long startTime) { + this.testClass = testClass; + this.testMethod = testMethod; + this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl"); + if (obsPath.toFile().exists()) { + obsPath.toFile().delete(); + } + this.startTime = startTime; + this.initializeTestCase(); + } + + public void initializeTestCase() { + testCaseJsonObject = objectMapper.createObjectNode(); + testCaseJsonObject.putObject("features"); + testCaseJsonObject.putObject("timing"); + testCaseJsonObject.putObject("coverage"); + testCaseJsonObject.putObject("args"); + testCaseJsonObject.putObject("metadata"); + testCaseJsonObject.put("type", "test_case"); + testCaseJsonObject.put("run_start", startTime); + testCaseJsonObject.put("property", testMethod); + } + + public static void event(String value, Object payload) throws RuntimeException { + // Add the payload to the features object + JsonNode jsonFeaturesNode = testCaseJsonObject.get("features"); + ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode; + + if (payload instanceof Integer) { + featuresNode.put(value, (Integer) payload); + } else if (payload instanceof String) { + featuresNode.put(value, (String) payload); + } else if (payload instanceof Float) { + featuresNode.put(value, (Float) payload); + } else { + throw new RuntimeException("Unsupported payload type for event"); + } + } + + public void addStatus(Result result) { + if (result == INVALID) { + testCaseJsonObject.put("status", "gave_up"); + testCaseJsonObject.put("status_reason", "assumption violated"); + } else if (result == FAILURE) { + testCaseJsonObject.put("status", "failed"); + testCaseJsonObject.put("status_reason", "Encountered exception"); + } else { + testCaseJsonObject.put("status", "passed"); + testCaseJsonObject.put("status_reason", ""); + } + + } + + public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) { + JsonNode timingNode = testCaseJsonObject.get("timing"); + ObjectNode timingObject = (ObjectNode) timingNode; + timingObject.put("generation", endGenerationTime - startTime); + timingObject.put("execution", endExecutionTime - endGenerationTime); + } + + public void add(String key, String value) { + testCaseJsonObject.put(key, value); + } + + public void writeToFile() { + // Append the JSON object to a file followed by a newline + try { + String jsonString = objectMapper.writeValueAsString(testCaseJsonObject); + try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) { + writer.write(jsonString); + writer.write(System.lineSeparator()); // Add a new line after each object + } + } catch (Exception e) { + throw new RuntimeException("Failed to write observations to file", e); + } + } +} From 05c3cd797c3d43ef9994fde03da382981bb6fad3 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Fri, 30 Aug 2024 13:27:41 -0400 Subject: [PATCH 06/14] Tyche event loggingg --- .../chocopy/ChocoPySemanticGeneratorTypeDirected.java | 6 +++--- .../cs/jqf/examples/chocopy/SemanticAnalysisTest.java | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java index 22dfc8bb6..646110d97 100644 --- a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -241,9 +241,9 @@ private String generateMethodCallExprOfType(SourceOfRandomness random, Type type } private String generateIndexExprOfType(SourceOfRandomness random, Type type) { - System.out.println("Generating index expr of type: " + type); +// System.out.println("Generating index expr of type: " + type); Type listType = new ListValueType(type); - System.out.println("Generating list expr of type: " + listType); +// System.out.println("Generating list expr of type: " + listType); String result = ""; result = generateListExprOfType(random, listType); String index = generateExpressionOfType(random, Type.INT_TYPE); @@ -273,7 +273,7 @@ private String generateCallExprOfType(SourceOfRandomness random, Type type) { } private String generateBinaryExprOfType(SourceOfRandomness random, Type type) { - System.out.println("Generating binary expr of type: " + type); +// System.out.println("Generating binary expr of type: " + type); Type operandType = null; String token = ""; if (type == Type.INT_TYPE) { diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java index 090ec6ef0..f501a325f 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java @@ -9,8 +9,7 @@ import edu.berkeley.cs.jqf.fuzz.JQF; import org.junit.runner.RunWith; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static edu.berkeley.cs.jqf.fuzz.util.Observability.event; import static org.junit.Assume.assumeTrue; @RunWith(JQF.class) @@ -21,7 +20,10 @@ public class SemanticAnalysisTest { public void fuzzSemanticAnalysis(@From(ChocoPySemanticGeneratorTypeDirected.class) String code) { Program program = RefParser.process(code, false); assumeTrue(!program.hasErrors()); + event("numStatements", program.statements.size()); + event("numDeclarations", program.declarations.size()); Program typedProgram = RefAnalysis.process(program); - assertFalse(typedProgram.hasErrors()); + event("numErrors", program.getErrorList().size()); + assumeTrue(!typedProgram.hasErrors()); } } From 4786c1fb11a460c801817cef6ee7910e3e517fc8 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 5 Sep 2024 10:54:11 -0400 Subject: [PATCH 07/14] Add args loggign --- .../cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java | 2 +- .../edu/berkeley/cs/jqf/fuzz/util/Observability.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 45d15baef..0d61919e4 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -186,7 +186,7 @@ public void evaluate() throws Throwable { if (result == SUCCESS) { observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); } - observability.add("representation", Arrays.toString(args)); + observability.addArgs(args); if (guidance instanceof ZestGuidance) { observability.add("how_generated", "Zest"); diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java index a8977948a..e3055bc47 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java @@ -8,6 +8,7 @@ import java.io.FileWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE; import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID; @@ -81,6 +82,15 @@ public void addTiming(long startTime, long endGenerationTime, long endExecutionT timingObject.put("execution", endExecutionTime - endGenerationTime); } + public void addArgs(Object[] args) { + JsonNode argsNode = testCaseJsonObject.get("args"); + ObjectNode argsObject = (ObjectNode) argsNode; + for (int i = 0; i < args.length; i++) { + argsObject.put("arg" + i, args[i].toString()); + } + add("representation", Arrays.toString(args)); + } + public void add(String key, String value) { testCaseJsonObject.put(key, value); } From 6309564c9574712b39a964c793546b6771cdf6d6 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 5 Sep 2024 12:45:23 -0400 Subject: [PATCH 08/14] Add observeGuidance --- .../edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java | 5 +++++ .../edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java | 8 ++++++++ .../edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java | 8 ++++++++ .../cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java | 11 +---------- .../edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java | 5 +++++ 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java index 9373689d5..11d540f41 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java @@ -167,6 +167,11 @@ public void finalize() { } } + @Override + public String observeGuidance() { + return "AFL"; + } + /** * Returns an input stream containing the bytes that AFL diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java index b9df383f5..bd79a454c 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java @@ -437,6 +437,14 @@ protected void appendLineToFile(File file, String line) throws GuidanceException } + @Override + public String observeGuidance() { + if (blind) { + return "Random"; + } + return "Zest"; + } + /* Writes a line of text to the log file. */ protected void infoLog(String str, Object... args) { if (verbose) { diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java index 45bfd2bc8..519bc6546 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java @@ -132,6 +132,14 @@ default void observeGeneratedArgs(Object[] args) { // Do nothing } + /** + *

This method is only invoked during optional observability + * logging to provide the type of guidance used for input generation.

+ */ + default String observeGuidance() { + return ""; + } + /** * Handles the end of a fuzzing trial. * diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 0d61919e4..b01d08d7c 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -187,16 +187,7 @@ public void evaluate() throws Throwable { observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); } observability.addArgs(args); - - if (guidance instanceof ZestGuidance) { - observability.add("how_generated", "Zest"); - } else if (guidance instanceof NoGuidance) { - observability.add("how_generated", "random"); - } else if (guidance instanceof ReproGuidance) { - observability.add("how_generated", "repro"); - } else { - observability.add("how_generated", "unknown"); - } + observability.add("how_generated", guidance.observeGuidance()); observability.writeToFile(); } diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java index e64e04554..64f85df4b 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java @@ -191,6 +191,11 @@ public void observeGeneratedArgs(Object[] args) { } } + @Override + public String observeGuidance() { + return "Repro"; + } + /** * Writes an object to a file * From 4797dd69c643788f13d87b2d1b3a9e46df903abc Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 12 Sep 2024 13:01:52 -0400 Subject: [PATCH 09/14] Initial type hierarchy --- .../ChocoPySemanticGeneratorTypeDirected.java | 65 ++++++++++++++----- .../chocopy/SemanticAnalysisTest.java | 3 +- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java index 646110d97..1e91f5f4f 100644 --- a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -43,6 +43,8 @@ public ChocoPySemanticGeneratorTypeDirected() { private static List classTypes; // Keeps track of all class types private static Map varTypes; // Keeps track of variables and their types private static Map funcTypes; // Keeps track of functions and their return types + private static Map parentClasses; // Keeps track of parent classes + private static Map> conformsSet; // Keeps track of conforming types private static int maxIdentifiers; private static int maxItems; private static int maxDepth; @@ -90,7 +92,6 @@ public ChocoPySemanticGeneratorTypeDirected() { Type.STR_TYPE, Type.BOOL_TYPE, Type.OBJECT_TYPE, - }; private static final String INDENT_TOKEN = " "; // 4 spaces @@ -108,11 +109,37 @@ public String generate(SourceOfRandomness random, GenerationStatus status) { this.allTypes = Arrays.asList(BASE_TYPES); this.varTypes = new HashMap<>(); this.funcTypes = new HashMap<>(); + this.parentClasses = new HashMap<>(); + this.conformsSet = new HashMap<>(); this.identCounter = 0; this.funcIdentCounter = 0; return generateProgram(random); } + public void initializeTypes() { + this.parentClasses.put() + } + + public void generateClassTypeHierarchy(SourceOfRandomness random) { + // Generate a random class type hierarchy. Start by generating a random number of classes, + // and then randomly decide which ones are subclasses. + int numClasses = nextIntBound(random, 1, maxBound, maxItems); + List classes = new ArrayList<>(); + for (int i = 0; i < numClasses; i++) { + classes.add(generateIdentifier(random)); + } + // Assign the parent-child relationships. Loop over the classes (after the first one), and randomly + // choose a parent from the previous elements. + for (int i = 1; i < numClasses; i++) { + if (random.nextBoolean()) { + int parentIndex = random.nextInt(0, i); + String parent = classes.get(parentIndex); + String child = classes.get(i); + parentClasses.put(new ClassValueType(child), new ClassValueType(parent)); + } + } + } + /** Utility method for generating a random list of items (e.g. statements, arguments, attributes) */ private static List generateItems(Function genMethod, SourceOfRandomness random, int minimum) { int len = nextIntBound(random, minimum, maxBound, maxItems); @@ -142,8 +169,8 @@ private static int nextIntBound(SourceOfRandomness random, int minimum, int maxi /** Generates a random ChocoPy program of classes, declarations, and statements */ private String generateProgram(SourceOfRandomness random) { String declarations = String.join("", generateItemsMultipleMethods(Arrays.asList( -// this::generateClassDef, - this::generateFuncDef, + this::generateClassDef, +// this::generateFuncDef, this::generateVarDef ), random, 0)); String statements = generateBlock(random, 0); @@ -245,7 +272,7 @@ private String generateIndexExprOfType(SourceOfRandomness random, Type type) { Type listType = new ListValueType(type); // System.out.println("Generating list expr of type: " + listType); String result = ""; - result = generateListExprOfType(random, listType); + result = generateExpressionOfType(random, listType); String index = generateExpressionOfType(random, Type.INT_TYPE); result += "[" + index + "]"; return result; @@ -306,7 +333,7 @@ private String generateBinaryExprOfType(SourceOfRandomness random, Type type) { nonPrimitiveTypes.add(new ListValueType(Type.INT_TYPE)); operandType = random.choose(nonPrimitiveTypes); if (operandType.isListType()) { - int listDepth = random.nextInt(0, maxDepth); + int listDepth = random.nextInt(1, maxDepth); operandType = random.choose(allTypes); for (int i = 0; i < listDepth; i++) { operandType = new ListValueType(operandType); @@ -329,7 +356,7 @@ private String generateBinaryExprOfType(SourceOfRandomness random, Type type) { } String lhs = generateExpressionOfType(random, operandType); String rhs = generateExpressionOfType(random, operandType); - return lhs + " " + token + " " + rhs; + return "(" + lhs + " " + token + " " + rhs + ")"; } private String generateUnaryExprOfType(SourceOfRandomness random, Type type) { @@ -365,6 +392,8 @@ private String generateListExprOfType(SourceOfRandomness random, Type type) { /** Generates a random ChocoPy expression using recursive calls */ private String generateExpressionOfType(SourceOfRandomness random, Type type) { +// System.out.println("Generating expression of type: " + type); +// System.out.println("Current expression depth: " + expressionDepth); String result; // Choose terminal if nesting depth is too high or based on a random flip of a coin int randDepth = nextIntBound(random, 0, maxBound, maxDepth); @@ -445,18 +474,18 @@ private String generateBlock(SourceOfRandomness random, int minimum) { return String.join("", generateItems(this::generateStatement, random, minimum)); } -// private String generateClassDef(SourceOfRandomness random) { -// String result = ""; -// String className = generateIdentifier(random); -// // Superclass could be one of the identifiers or object. Index should be from 0 to maxIdentifiers inclusive. -// int superClassIndex = nextIntBound(random, 0, classTypes.size(), maxIdentifiers); -// String superClassName = classTypes.get(superClassIndex); -// result += "class " + className + "(" + superClassName + "):\n"; -// indentLevel++; -// result += generateDeclarationBlock(random, 1); -// indentLevel--; -// return result + "\n"; -// } + private String generateClassDef(SourceOfRandomness random) { + String result = ""; + String className = generateIdentifier(random); + // Superclass could be one of the identifiers or object. Index should be from 0 to maxIdentifiers inclusive. + int superClassIndex = nextIntBound(random, 0, classTypes.size(), maxIdentifiers); + String superClassName = classTypes.get(superClassIndex); + result += "class " + className + "(" + superClassName + "):\n"; + indentLevel++; + result += generateDeclarationBlock(random, 1); + indentLevel--; + return result + "\n"; + } /** Generates a block of VarDefs and FuncDefs*/ private String generateDeclarationBlock(SourceOfRandomness random, int minimum) { diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java index f501a325f..3dfbe3332 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java @@ -10,6 +10,7 @@ import org.junit.runner.RunWith; import static edu.berkeley.cs.jqf.fuzz.util.Observability.event; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @RunWith(JQF.class) @@ -24,6 +25,6 @@ public void fuzzSemanticAnalysis(@From(ChocoPySemanticGeneratorTypeDirected.clas event("numDeclarations", program.declarations.size()); Program typedProgram = RefAnalysis.process(program); event("numErrors", program.getErrorList().size()); - assumeTrue(!typedProgram.hasErrors()); + assertTrue(!typedProgram.hasErrors()); } } From 6ab346f29faa581dd0163d45e5eb4170f26f801c Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Wed, 18 Sep 2024 15:18:34 -0400 Subject: [PATCH 10/14] Added classdef --- .../ChocoPySemanticGeneratorTypeDirected.java | 126 +++++++++++++++--- .../chocopy/SemanticAnalysisTest.java | 10 ++ 2 files changed, 117 insertions(+), 19 deletions(-) diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java index 1e91f5f4f..45fbfc291 100644 --- a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -9,7 +9,6 @@ import java.util.*; import java.util.function.Function; import org.apache.commons.lang3.tuple.Pair; -import chocopy.common.astnodes.ClassType; import chocopy.common.analysis.types.*; import static org.junit.Assume.assumeFalse; @@ -42,8 +41,10 @@ public ChocoPySemanticGeneratorTypeDirected() { private static List allTypes; // Keeps track of all types private static List classTypes; // Keeps track of all class types private static Map varTypes; // Keeps track of variables and their types + private static Map> attrTypes; // Keeps track of attributes and their types private static Map funcTypes; // Keeps track of functions and their return types private static Map parentClasses; // Keeps track of parent classes + private static Map> childClasses; // Keeps track of child classes private static Map> conformsSet; // Keeps track of conforming types private static int maxIdentifiers; private static int maxItems; @@ -108,38 +109,81 @@ public String generate(SourceOfRandomness random, GenerationStatus status) { this.classTypes = new ArrayList<>(); this.allTypes = Arrays.asList(BASE_TYPES); this.varTypes = new HashMap<>(); + this.attrTypes = new HashMap<>(); this.funcTypes = new HashMap<>(); this.parentClasses = new HashMap<>(); + this.childClasses = new HashMap<>(); + this.childClasses.put(Type.OBJECT_TYPE, new ArrayList<>()); this.conformsSet = new HashMap<>(); this.identCounter = 0; this.funcIdentCounter = 0; return generateProgram(random); } - public void initializeTypes() { - this.parentClasses.put() - } - public void generateClassTypeHierarchy(SourceOfRandomness random) { // Generate a random class type hierarchy. Start by generating a random number of classes, // and then randomly decide which ones are subclasses. - int numClasses = nextIntBound(random, 1, maxBound, maxItems); + int numClasses = nextIntBound(random, 0, maxBound, maxItems); + if (numClasses == 0) { + return; + } List classes = new ArrayList<>(); for (int i = 0; i < numClasses; i++) { - classes.add(generateIdentifier(random)); + String classIdent = generateIdentifier(random); + classes.add(classIdent); + ValueType classType = new ClassValueType(classIdent); + classTypes.add(classType); } // Assign the parent-child relationships. Loop over the classes (after the first one), and randomly // choose a parent from the previous elements. + Type firstClassType = new ClassValueType(classes.get(0)); + childClasses.get(Type.OBJECT_TYPE).add(firstClassType); + parentClasses.put(firstClassType, Type.OBJECT_TYPE); for (int i = 1; i < numClasses; i++) { if (random.nextBoolean()) { int parentIndex = random.nextInt(0, i); String parent = classes.get(parentIndex); String child = classes.get(i); - parentClasses.put(new ClassValueType(child), new ClassValueType(parent)); + Type parentType = new ClassValueType(parent); + Type childType = new ClassValueType(child); + parentClasses.put(childType, parentType); + if (!childClasses.containsKey(parentType)) { + childClasses.put(parentType, new ArrayList<>()); + } + childClasses.get(parentType).add(childType); + } else { + Type classType = new ClassValueType(classes.get(i)); + parentClasses.put(classType, Type.OBJECT_TYPE); + childClasses.get(Type.OBJECT_TYPE).add(classType); } } } + public Type generateConformingType(SourceOfRandomness random, Type type) { + if (type.isSpecialType()) { + return type; + } else if (type.isListType()) { + float randFloat = random.nextFloat(); + if (randFloat > 0.75) { + return type; + } else { + return Type.EMPTY_TYPE; + } + } else if (type.isValueType()) { + if (!childClasses.containsKey(type)) { + return type; + } else { + List subTypes = childClasses.get(type); + subTypes.add(type); + return random.choose(subTypes); + } + } else { + // Should not be function type + assert(false); + } + return Type.EMPTY_TYPE; + } + /** Utility method for generating a random list of items (e.g. statements, arguments, attributes) */ private static List generateItems(Function genMethod, SourceOfRandomness random, int minimum) { int len = nextIntBound(random, minimum, maxBound, maxItems); @@ -168,27 +212,32 @@ private static int nextIntBound(SourceOfRandomness random, int minimum, int maxi /** Generates a random ChocoPy program of classes, declarations, and statements */ private String generateProgram(SourceOfRandomness random) { + String classDeclarations = ""; + generateClassTypeHierarchy(random); + for (Type classType : classTypes) { + classDeclarations += generateClassDefOfType(random, classType); + } + String declarations = String.join("", generateItemsMultipleMethods(Arrays.asList( - this::generateClassDef, // this::generateFuncDef, this::generateVarDef ), random, 0)); String statements = generateBlock(random, 0); - return declarations + statements; + return classDeclarations + declarations + statements; } /** Generates a random ChocoPy declaration */ - private String generateDeclaration(SourceOfRandomness random) { + private String generateDeclaration(SourceOfRandomness random, Type classType) { String result = StringUtils.repeat(INDENT_TOKEN, indentLevel); int randDepth = nextIntBound(random, 0, maxBound, maxDepth); if (declarationDepth >= randDepth) { // Choose a random private method from this class, and then call it with `random` - result += generateVarDef(random); + result += generateAttrDef(random, classType); } else { // If depth is low and we won the flip, then generate compound declarations // (that is, declarations that contain other declarations) result += random.choose(Arrays.>asList( - this::generateVarDef + r -> generateAttrDef(r, classType) // this::generateFuncDef )).apply(random); } @@ -425,6 +474,10 @@ private String generateExpressionOfType(SourceOfRandomness random, Type type) { r -> generateCallExprOfType(r, type), r -> generateBinaryExprOfType(r, type) )).apply(random); + } else if (type == Type.EMPTY_TYPE) { + return "[]"; + } else if (type == Type.NONE_TYPE) { + return "None"; } else { result = random.choose(Arrays.>asList( r -> generateConstExprOfType(r, type), @@ -449,8 +502,14 @@ private String generateConstExprOfType(SourceOfRandomness random, Type type) { return generateStringLiteral(random); } else if (type == Type.BOOL_TYPE) { return generateBoolLiteral(random); + } else if (type == Type.EMPTY_TYPE) { + return "[]"; + } else if (type == Type.NONE_TYPE) { + return "None"; } else if (type.isListType()) { return "[" + generateConstExprOfType(random, type.elementType()) + "]"; + } else if (type.isValueType()) { + return type.className() + "()"; } else { assumeTrue(false); return ""; @@ -466,7 +525,12 @@ private String generateAssignStmt(SourceOfRandomness random) { Type varType = varTypes.get(var); result += var + " = "; - return result + generateExpressionOfType(random, varType); + // Randomly generate a conforming type +// System.out.println("Generating expression of type: " + varType); + Type conformingType = generateConformingType(random, varType); +// System.out.println("Generating conforming expression of type: " + conformingType); + + return result + generateExpressionOfType(random, conformingType); } /** Generates a block of statements, excluding the statement */ @@ -479,17 +543,28 @@ private String generateClassDef(SourceOfRandomness random) { String className = generateIdentifier(random); // Superclass could be one of the identifiers or object. Index should be from 0 to maxIdentifiers inclusive. int superClassIndex = nextIntBound(random, 0, classTypes.size(), maxIdentifiers); - String superClassName = classTypes.get(superClassIndex); + String superClassName = generateIdentifier(random); result += "class " + className + "(" + superClassName + "):\n"; indentLevel++; - result += generateDeclarationBlock(random, 1); + result += generateDeclarationBlock(random, null, 1); + indentLevel--; + return result + "\n"; + } + + private String generateClassDefOfType(SourceOfRandomness random, Type type) { + String result = ""; + // Superclass could be one of the identifiers or object. Index should be from 0 to maxIdentifiers inclusive. + Type superClassType = parentClasses.get(type); + result += "class " + type.className() + "(" + superClassType.className() + "):\n"; + indentLevel++; + result += generateDeclarationBlock(random, type, 1); indentLevel--; return result + "\n"; } /** Generates a block of VarDefs and FuncDefs*/ - private String generateDeclarationBlock(SourceOfRandomness random, int minimum) { - return String.join("", generateItems(this::generateDeclaration, random, minimum)); + private String generateDeclarationBlock(SourceOfRandomness random, Type classType, int minimum) { + return String.join("", generateItems(r -> generateDeclaration(r, classType), random, minimum)); } private String generateExpressionStmt(SourceOfRandomness random) { @@ -679,7 +754,6 @@ private ValueType generateListType(SourceOfRandomness random) { private Pair generateTypedVar(SourceOfRandomness random) { ValueType type = generateType(random); String ident = generateIdentifier(random); - varTypes.put(ident, type); return Pair.of(ident, type); } @@ -687,6 +761,20 @@ private String generateVarDef(SourceOfRandomness random) { Pair typedVar = generateTypedVar(random); String ident = typedVar.getLeft(); ValueType varType = typedVar.getRight(); + varTypes.put(ident, varType); + + return ident + ":" + varType + " = " + generateLiteralOfType(random, varType) + "\n"; + } + + private String generateAttrDef(SourceOfRandomness random, Type classType) { + Pair typedVar = generateTypedVar(random); + String ident = typedVar.getLeft(); + ValueType varType = typedVar.getRight(); + + if (!attrTypes.containsKey(classType)) { + attrTypes.put(classType, new HashMap<>()); + } + attrTypes.get(classType).put(ident, varType); return ident + ":" + varType + " = " + generateLiteralOfType(random, varType) + "\n"; } diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java index 3dfbe3332..43dcd4a69 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java @@ -3,6 +3,7 @@ import chocopy.ChocoPy; import chocopy.common.astnodes.Program; import chocopy.reference.RefAnalysis; +import chocopy.reference.RefCodeGen; import chocopy.reference.RefParser; import com.pholser.junit.quickcheck.From; import edu.berkeley.cs.jqf.fuzz.Fuzz; @@ -27,4 +28,13 @@ public void fuzzSemanticAnalysis(@From(ChocoPySemanticGeneratorTypeDirected.clas event("numErrors", program.getErrorList().size()); assertTrue(!typedProgram.hasErrors()); } + + @Fuzz + public void fuzzCodeGen(@From(ChocoPySemanticGenerator.class) String code) { + Program program = RefParser.process(code, false); + assumeTrue(!program.hasErrors()); + Program typedProgram = RefAnalysis.process(program); + assumeTrue(!typedProgram.hasErrors()); + RefCodeGen.process(typedProgram); + } } From 3a137b14cef67a136ad8dd1f5c1beb21f9b78aed Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 19 Sep 2024 15:24:36 -0400 Subject: [PATCH 11/14] Added helper scopes, method calls, nonlocal + global --- .../ChocoPySemanticGeneratorTypeDirected.java | 253 +++++++++++------- .../cs/jqf/examples/chocopy/Scope.java | 96 +++++++ 2 files changed, 254 insertions(+), 95 deletions(-) create mode 100644 examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/Scope.java diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java index 45fbfc291..1d9fb0b06 100644 --- a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -1,5 +1,6 @@ package edu.berkeley.cs.jqf.examples.chocopy; +import com.pholser.junit.quickcheck.Pair; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.random.SourceOfRandomness; @@ -8,7 +9,6 @@ import java.util.*; import java.util.function.Function; -import org.apache.commons.lang3.tuple.Pair; import chocopy.common.analysis.types.*; import static org.junit.Assume.assumeFalse; @@ -40,12 +40,12 @@ public ChocoPySemanticGeneratorTypeDirected() { private static List funcIdentifiers; // Stores generated IDs, to promote re-use private static List allTypes; // Keeps track of all types private static List classTypes; // Keeps track of all class types - private static Map varTypes; // Keeps track of variables and their types private static Map> attrTypes; // Keeps track of attributes and their types - private static Map funcTypes; // Keeps track of functions and their return types + private static Map> methodTypes; // Keeps track of methods and their types private static Map parentClasses; // Keeps track of parent classes private static Map> childClasses; // Keeps track of child classes - private static Map> conformsSet; // Keeps track of conforming types + private static Scope globalScope; // Keeps track of global scope + private Scope currentScope; // Keeps track of current scope private static int maxIdentifiers; private static int maxItems; private static int maxDepth; @@ -108,13 +108,13 @@ public String generate(SourceOfRandomness random, GenerationStatus status) { this.indentLevel = 0; this.classTypes = new ArrayList<>(); this.allTypes = Arrays.asList(BASE_TYPES); - this.varTypes = new HashMap<>(); this.attrTypes = new HashMap<>(); - this.funcTypes = new HashMap<>(); + this.methodTypes = new HashMap<>(); this.parentClasses = new HashMap<>(); this.childClasses = new HashMap<>(); this.childClasses.put(Type.OBJECT_TYPE, new ArrayList<>()); - this.conformsSet = new HashMap<>(); + this.globalScope = new Scope("global", null); + this.currentScope = globalScope; this.identCounter = 0; this.funcIdentCounter = 0; return generateProgram(random); @@ -219,28 +219,20 @@ private String generateProgram(SourceOfRandomness random) { } String declarations = String.join("", generateItemsMultipleMethods(Arrays.asList( -// this::generateFuncDef, + this::generateFuncDef, this::generateVarDef ), random, 0)); - String statements = generateBlock(random, 0); + String statements = generateBlock(random, 1); return classDeclarations + declarations + statements; } /** Generates a random ChocoPy declaration */ - private String generateDeclaration(SourceOfRandomness random, Type classType) { + private String generateClassDeclaration(SourceOfRandomness random, Type classType) { String result = StringUtils.repeat(INDENT_TOKEN, indentLevel); - int randDepth = nextIntBound(random, 0, maxBound, maxDepth); - if (declarationDepth >= randDepth) { - // Choose a random private method from this class, and then call it with `random` - result += generateAttrDef(random, classType); - } else { - // If depth is low and we won the flip, then generate compound declarations - // (that is, declarations that contain other declarations) - result += random.choose(Arrays.>asList( - r -> generateAttrDef(r, classType) -// this::generateFuncDef - )).apply(random); - } + result += random.choose(Arrays.>asList( + r -> generateAttrDef(r, classType), + r -> generateMethodDef(r, classType) + )).apply(random); return result; } @@ -251,9 +243,9 @@ private String generateFuncDeclaration(SourceOfRandomness random) { if (declarationDepth >= randDepth) { // Choose a random private method from this class, and then call it with `random` result += random.choose(Arrays.>asList( - this::generateVarDef -// this::generateNonlocalDecl, -// this::generateGlobalDecl + this::generateVarDef, + this::generateNonlocalDecl, + this::generateGlobalDecl )).apply(random); } else { // If depth is low and we won the flip, then generate compound declarations @@ -283,6 +275,9 @@ private String generateStatement(SourceOfRandomness random) { // If depth is low and we won the flip, then generate compound statements // (that is, statements that contain other statements) result += random.choose(Arrays.>asList( + this::generateAssignStmt, + this::generatePassStmt, + this::generateExpressionStmt, this::generateIfStmt, this::generateForStmt, this::generateWhileStmt @@ -291,13 +286,8 @@ private String generateStatement(SourceOfRandomness random) { return result + "\n"; } - private String generateVarOfType(SourceOfRandomness random, Type type) { - List candidateVars = new ArrayList<>(); - for (Map.Entry entry : varTypes.entrySet()) { - if (entry.getValue().equals(type)) { - candidateVars.add(entry.getKey()); - } - } + private String generateVarOfType(SourceOfRandomness random, Type type, boolean onlyCurrentScope) { + List candidateVars = currentScope.getVarsOfType(type, onlyCurrentScope); assumeTrue(!candidateVars.isEmpty()); return random.choose(candidateVars); } @@ -308,12 +298,36 @@ private String generateParenExprOfType(SourceOfRandomness random, Type type) { // TODO: implement private String generateMemberExprOfType(SourceOfRandomness random, Type type) { - return ""; + List> attrCandidates = new ArrayList<>(); + for (Type classType : attrTypes.keySet()) { + Map attrs = attrTypes.get(classType); + for (String attr : attrs.keySet()) { + if (attrs.get(attr).equals(type)) { + attrCandidates.add(new Pair(classType, attr)); + } + } + } + assumeTrue(!attrCandidates.isEmpty()); + Pair attrPair = random.choose(attrCandidates); + Type classType = generateConformingType(random, attrPair.first); + String objectExpr = generateExpressionOfType(random, classType); + String attr = attrPair.second; + return objectExpr + "." + attr; } // TODO: implement private String generateMethodCallExprOfType(SourceOfRandomness random, Type type) { - return ""; + // Need to find a method that returns the correct type + List> candidateFuncs = globalScope.getMethodsWithReturnType(type); + assumeTrue(!candidateFuncs.isEmpty()); + Pair func = random.choose(candidateFuncs); + FuncType funcType = func.second; + List paramTypes = funcType.parameters; + List args = new ArrayList<>(); + for (Type paramType : paramTypes) { + args.add(generateExpressionOfType(random, generateConformingType(random, paramType))); + } + return func + "(" + String.join(", ", args) + ")"; } private String generateIndexExprOfType(SourceOfRandomness random, Type type) { @@ -329,21 +343,14 @@ private String generateIndexExprOfType(SourceOfRandomness random, Type type) { private String generateCallExprOfType(SourceOfRandomness random, Type type) { // Need to find a function that returns the correct type - List candidateFuncs = new ArrayList<>(); - for (Map.Entry entry : funcTypes.entrySet()) { - Type returnType = entry.getValue().returnType; - if (returnType.equals(type)) { - candidateFuncs.add(entry.getKey()); - } - } + List> candidateFuncs = currentScope.getFuncsWithReturnType(type); assumeTrue(!candidateFuncs.isEmpty()); - int funcIndex = random.nextInt(0, candidateFuncs.size()); - String func = candidateFuncs.get(funcIndex); - FuncType funcType = funcTypes.get(func); - List paramTypes = funcTypes.get(func).parameters; + Pair func = random.choose(candidateFuncs); + FuncType funcType = func.second; + List paramTypes = funcType.parameters; List args = new ArrayList<>(); for (Type paramType : paramTypes) { - args.add(generateExpressionOfType(random, paramType)); + args.add(generateExpressionOfType(random, generateConformingType(random, paramType))); } return func + "(" + String.join(", ", args) + ")"; } @@ -420,17 +427,17 @@ private String generateUnaryExprOfType(SourceOfRandomness random, Type type) { } else { assert(false); } - return token + " " + generateExpressionOfType(random, operandType); + return "(" + token + " " + generateExpressionOfType(random, operandType) + ")"; } private String generateListExprOfType(SourceOfRandomness random, Type type) { assert(type.isListType()); String result = random.choose(Arrays.>asList( r -> generateConstExprOfType(r, type), - r -> generateVarOfType(r, type), + r -> generateVarOfType(r, type, false), r -> generateParenExprOfType(r, type), -// r -> generateMemberExprOfType(r, type), -// r -> generateMethodCallExprOfType(r, type), + r -> generateMemberExprOfType(r, type), + r -> generateMethodCallExprOfType(r, type), r -> generateIndexExprOfType(r, type), r -> generateCallExprOfType(r, type), r -> generateBinaryExprOfType(r, type) @@ -454,10 +461,10 @@ private String generateExpressionOfType(SourceOfRandomness random, Type type) { if (type == Type.INT_TYPE || type == Type.BOOL_TYPE) { result = random.choose(Arrays.>asList( r -> generateConstExprOfType(r, type), - r -> generateVarOfType(r, type), + r -> generateVarOfType(r, type, false), r -> generateParenExprOfType(r, type), -// r -> generateMemberExprOfType(r, type), -// r -> generateMethodCallExprOfType(r, type), + r -> generateMemberExprOfType(r, type), + r -> generateMethodCallExprOfType(r, type), r -> generateIndexExprOfType(r, type), r -> generateCallExprOfType(r, type), r -> generateBinaryExprOfType(r, type), @@ -466,10 +473,10 @@ private String generateExpressionOfType(SourceOfRandomness random, Type type) { } else if (type.isListType() || type == Type.STR_TYPE) { result = random.choose(Arrays.>asList( r -> generateConstExprOfType(r, type), - r -> generateVarOfType(r, type), + r -> generateVarOfType(r, type, false), r -> generateParenExprOfType(r, type), -// r -> generateMemberExprOfType(r, type), -// r -> generateMethodCallExprOfType(r, type), + r -> generateMemberExprOfType(r, type), + r -> generateMethodCallExprOfType(r, type), r -> generateIndexExprOfType(r, type), r -> generateCallExprOfType(r, type), r -> generateBinaryExprOfType(r, type) @@ -481,10 +488,10 @@ private String generateExpressionOfType(SourceOfRandomness random, Type type) { } else { result = random.choose(Arrays.>asList( r -> generateConstExprOfType(r, type), - r -> generateVarOfType(r, type), + r -> generateVarOfType(r, type, false), r -> generateParenExprOfType(r, type), -// r -> generateMemberExprOfType(r, type), -// r -> generateMethodCallExprOfType(r, type), + r -> generateMemberExprOfType(r, type), + r -> generateMethodCallExprOfType(r, type), r -> generateIndexExprOfType(r, type), r -> generateCallExprOfType(r, type) )).apply(random); @@ -519,10 +526,11 @@ private String generateConstExprOfType(SourceOfRandomness random, Type type) { // TODO: Add member expressions and method calls private String generateAssignStmt(SourceOfRandomness random) { String result = ""; - // Choose a random entry in varTypes - assumeTrue(!varTypes.isEmpty()); - String var = random.choose(varTypes.entrySet()).getKey(); - Type varType = varTypes.get(var); + // Choose a random entry in varTypes. NOTE: varTypes will include variables from parent scopes + // if they were declared nonlocal or global + assumeTrue(!currentScope.varTypes.isEmpty()); + String var = random.choose(currentScope.varTypes.entrySet()).getKey(); + Type varType = currentScope.varTypes.get(var); result += var + " = "; // Randomly generate a conforming type @@ -546,7 +554,7 @@ private String generateClassDef(SourceOfRandomness random) { String superClassName = generateIdentifier(random); result += "class " + className + "(" + superClassName + "):\n"; indentLevel++; - result += generateDeclarationBlock(random, null, 1); + result += generateClassDeclarationBlock(random, null); indentLevel--; return result + "\n"; } @@ -557,14 +565,15 @@ private String generateClassDefOfType(SourceOfRandomness random, Type type) { Type superClassType = parentClasses.get(type); result += "class " + type.className() + "(" + superClassType.className() + "):\n"; indentLevel++; - result += generateDeclarationBlock(random, type, 1); + result += generateClassDeclarationBlock(random, type); indentLevel--; return result + "\n"; } /** Generates a block of VarDefs and FuncDefs*/ - private String generateDeclarationBlock(SourceOfRandomness random, Type classType, int minimum) { - return String.join("", generateItems(r -> generateDeclaration(r, classType), random, minimum)); + private String generateClassDeclarationBlock(SourceOfRandomness random, Type classType) { + String declarations = String.join("", generateItems(r -> generateClassDeclaration(r, classType), random, 1)); + return String.join("", declarations); } private String generateExpressionStmt(SourceOfRandomness random) { @@ -575,7 +584,7 @@ private String generateExpressionStmt(SourceOfRandomness random) { private String generateForStmt(SourceOfRandomness random) { statementDepth++; ValueType listType = generateListType(random); - String var = generateVarOfType(random, listType.elementType()); + String var = generateVarOfType(random, listType.elementType(), true); String s = "for " + var + " in " + generateExpressionOfType(random, listType) + ":\n"; indentLevel++; s += generateBlock(random, 1); @@ -588,14 +597,19 @@ private String generateFuncDef(SourceOfRandomness random) { declarationDepth++; String funcIdent = generateFuncIdentifier(random); + currentScope = new Scope(funcIdent, currentScope); int numParams = nextIntBound(random, 0, maxItems, maxItems); StringBuilder result = new StringBuilder("def " + generateFuncIdentifier(random) + "("); List paramTypes = new ArrayList<>(); + List paramNames = new ArrayList<>(); for (int i = 0; i < numParams; i++) { Pair param = generateTypedVar(random); - paramTypes.add(param.getRight()); - result.append(param.getLeft()).append(":").append(param.getRight()); + paramTypes.add(param.second); + paramNames.add(param.first + ":" + param.second); + currentScope.varTypes.put(param.first, param.second); } + result.append(String.join(",", paramNames)); + result.append(")"); ValueType returnType = Type.NONE_TYPE; @@ -604,25 +618,75 @@ private String generateFuncDef(SourceOfRandomness random) { result.append("->").append(returnType); } FuncType funcType = new FuncType(paramTypes, returnType); - result.append(")"); + result.append(":\n"); + indentLevel++; + result.append(String.join("", generateItems(this::generateFuncDeclaration, random, 0))); + result.append(generateBlock(random, 1)); + if (returnType != Type.NONE_TYPE) { + result.append(StringUtils.repeat(INDENT_TOKEN, indentLevel)).append(generateReturnStmtOfType(random, generateConformingType(random, returnType))); + } + currentScope.funcTypes.put(funcIdent, funcType); + indentLevel--; + declarationDepth--; + currentScope = currentScope.getParent(); + return result + "\n"; + } + + private String generateMethodDef(SourceOfRandomness random, Type classType) { + declarationDepth++; + String funcIdent = generateFuncIdentifier(random); + currentScope = new Scope(funcIdent, currentScope); + int numParams = nextIntBound(random, 0, maxItems, maxItems); + String firstParam = generateIdentifier(random); + ValueType firstParamType = new ClassValueType("\"" + classType.className() + "\""); + StringBuilder result = new StringBuilder("def " + generateFuncIdentifier(random) + "("); + List paramTypes = new ArrayList<>(); + List paramNames = new ArrayList<>(); + paramTypes.add(firstParamType); + paramNames.add(firstParam + ":" + firstParamType); + for (int i = 1; i < numParams; i++) { + Pair param = generateTypedVar(random); + paramTypes.add(param.second); + paramNames.add(param.first + ":" + param.second); + currentScope.varTypes.put(param.first, param.second); + } + result.append(String.join(",", paramNames)); + result.append(")"); - funcTypes.put(funcIdent, funcType); + ValueType returnType = Type.NONE_TYPE; + if (random.nextBoolean()) { + returnType = generateType(random); + result.append("->").append(returnType); + } + FuncType funcType = new FuncType(paramTypes, returnType); result.append(":\n"); indentLevel++; result.append(String.join("", generateItems(this::generateFuncDeclaration, random, 0))); result.append(generateBlock(random, 1)); if (returnType != Type.NONE_TYPE) { - result.append(generateReturnStmtOfType(random, returnType)); + result.append(StringUtils.repeat(INDENT_TOKEN, indentLevel)).append(generateReturnStmtOfType(random, generateConformingType(random, returnType))); } + if (!methodTypes.containsKey(classType)) { + methodTypes.put(classType, new HashMap<>()); + } + methodTypes.get(classType).put(funcIdent, funcType); + currentScope.funcTypes.put(classType + "." + funcIdent, funcType); indentLevel--; declarationDepth--; + currentScope = currentScope.getParent(); return result + "\n"; } private String generateGlobalDecl(SourceOfRandomness random) { - return "global " + generateIdentifier(random) + "\n"; + assumeTrue(!currentScope.name.equals("global")); + assumeTrue(!globalScope.varTypes.isEmpty()); + String var = random.choose(globalScope.varTypes.keySet()); + assumeTrue(!currentScope.varTypes.containsKey(var)); + Type varType = globalScope.varTypes.get(var); + currentScope.varTypes.put(var, varType); + return "global " + var + "\n"; } private String generateIdentifier(SourceOfRandomness random) { @@ -706,18 +770,14 @@ private String generateNoneLiteral(SourceOfRandomness random) { return "None"; } -// private String generateMemberExpr(SourceOfRandomness random) { -// return "(" + generateCExpression(random) + ")." + generateIdentifier(random); -// } -// -// private String generateMethodCallExpr(SourceOfRandomness random) { -// return generateCExpression(random) + "." + generateCallExpr(random); -// } -// -// private String generateNonlocalDecl(SourceOfRandomness random) { -// return "nonlocal " + generateIdentifier(random) + "\n"; -// } -// + private String generateNonlocalDecl(SourceOfRandomness random) { + List> candidateVars = currentScope.getNonlocalVars(globalScope); + assumeTrue(!candidateVars.isEmpty()); + Pair var = random.choose(candidateVars); + currentScope.varTypes.put(var.first, var.second); + return "nonlocal " + var.first + "\n"; + } + private String generatePassStmt(SourceOfRandomness random) { return "pass"; @@ -729,11 +789,14 @@ private String generateReturnStmtOfType(SourceOfRandomness random, Type type) { /** Randomly choose from types and random list depth using maxDepth parameter */ private ValueType generateType(SourceOfRandomness random) { - ValueType baseType = random.choose(allTypes); - int listDepth = random.nextInt(0, maxDepth); - if (listDepth == 0) { + List candidateTypes = new ArrayList<>(allTypes); + candidateTypes.addAll(classTypes); + ValueType baseType = (ValueType) random.choose(candidateTypes); + float randFloat = random.nextFloat(); + if (randFloat > 0.75) { return baseType; } else { + int listDepth = random.nextInt(0, maxDepth); for (int i = 0; i < listDepth; i++) { baseType = new ListValueType(baseType); } @@ -754,22 +817,22 @@ private ValueType generateListType(SourceOfRandomness random) { private Pair generateTypedVar(SourceOfRandomness random) { ValueType type = generateType(random); String ident = generateIdentifier(random); - return Pair.of(ident, type); + return new Pair(ident, type); } private String generateVarDef(SourceOfRandomness random) { Pair typedVar = generateTypedVar(random); - String ident = typedVar.getLeft(); - ValueType varType = typedVar.getRight(); - varTypes.put(ident, varType); + String ident = typedVar.first; + ValueType varType = typedVar.second; + currentScope.varTypes.put(ident, varType); return ident + ":" + varType + " = " + generateLiteralOfType(random, varType) + "\n"; } private String generateAttrDef(SourceOfRandomness random, Type classType) { Pair typedVar = generateTypedVar(random); - String ident = typedVar.getLeft(); - ValueType varType = typedVar.getRight(); + String ident = typedVar.first; + ValueType varType = typedVar.second; if (!attrTypes.containsKey(classType)) { attrTypes.put(classType, new HashMap<>()); diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/Scope.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/Scope.java new file mode 100644 index 000000000..36855c928 --- /dev/null +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/Scope.java @@ -0,0 +1,96 @@ +package edu.berkeley.cs.jqf.examples.chocopy; + +import chocopy.common.analysis.types.FuncType; +import chocopy.common.analysis.types.Type; +import com.pholser.junit.quickcheck.Pair; + +import java.sql.Array; +import java.util.*; + +public class Scope { + + public String name; + private Scope parentScope; + private List childScopes; + public Map varTypes; + public Map funcTypes; + + public Scope(String name, Scope parentScope) { + this.name = name; + this.parentScope = parentScope; + this.childScopes = new ArrayList<>(); + this.varTypes = new HashMap<>(); + this.funcTypes = new HashMap<>(); + } + + public Scope getParent() { + return parentScope; + } + + public void addChild(Scope child) { + childScopes.add(child); + } + + public List> getNonlocalVars(Scope globalScope) { + List> nonlocalVars = new ArrayList<>(); + if (parentScope == null || parentScope.name.equals("global")) { + return nonlocalVars; + } + for (Map.Entry entry : parentScope.varTypes.entrySet()) { + String varName = entry.getKey(); + if (varTypes.containsKey(varName) || globalScope.varTypes.containsKey(varName)) { + continue; + } + nonlocalVars.add(new Pair<>(entry.getKey(), entry.getValue())); + } + return nonlocalVars; + } + + public List getVarsOfType(Type type, boolean onlyCurrentScope) { + List varTypesList = new ArrayList<>(); + for (Map.Entry entry : varTypes.entrySet()) { + if (entry.getValue().equals(type)) { + varTypesList.add(entry.getKey()); + } + } + if (onlyCurrentScope || parentScope == null) { + return varTypesList; + } + varTypesList.addAll(parentScope.getVarsOfType(type, false)); + return varTypesList; + } + + public List> getFuncsWithReturnType(Type type) { + List> funcTypesList = new ArrayList<>(); + for (Map.Entry entry : funcTypes.entrySet()) { + FuncType funcType = entry.getValue(); + if (funcType.returnType.equals(type)) { + funcTypesList.add(new Pair(entry.getKey(), funcType)); + } + } + if (parentScope == null) { + return funcTypesList; + } + funcTypesList.addAll(parentScope.getFuncsWithReturnType(type)); + return funcTypesList; + } + + public List> getMethodsWithReturnType(Type type) { + List> funcTypesList = new ArrayList<>(); + for (Map.Entry entry : funcTypes.entrySet()) { + String funcName = entry.getKey(); + if (!funcName.contains(".")) { + continue; + } + FuncType funcType = entry.getValue(); + if (funcType.returnType.equals(type)) { + funcTypesList.add(new Pair<>(entry.getKey(), funcType)); + } + } + if (parentScope == null) { + return funcTypesList; + } + funcTypesList.addAll(parentScope.getFuncsWithReturnType(type)); + return funcTypesList; + } +} From eb9957a78942db84b1b64f840f1cfafe43863ab4 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 19 Sep 2024 15:42:03 -0400 Subject: [PATCH 12/14] GPT generated (and fixed) comment summarizing generation strategy --- .../ChocoPySemanticGeneratorTypeDirected.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java index 1d9fb0b06..29e08fc46 100644 --- a/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java +++ b/examples/src/main/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPySemanticGeneratorTypeDirected.java @@ -14,7 +14,79 @@ import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; -/* Generates random strings that are syntactically valid ChocoPy */ +/** + * ChocoPy Program Generator: Detailed Summary + * + * The ChocoPy program generator is designed to produce random, syntactically, and semantically valid + * ChocoPy programs using a recursive, type-driven generation strategy. Here's a breakdown of the core + * principles and methodology used in the generator: + * + * General Strategy: + * + * 1. Class Type Hierarchy Generation: + * - A random number of class types are generated. + * - Establishes parent-child relationships between classes, ensuring that every class ultimately descends + * from the base Object type. + * + * 2. Type Conformance: + * - Ensures that generated expressions and statements are type-conformant by using predefined and dynamically + * generated types. + * - Selects types that respect the established class hierarchy and type relationships. + * + * Program Generation: + * + * 1. Program Structure: + * - The top-level `generateProgram` function initializes class hierarchies and assembles classes, declarations, + * and statements into a complete program. + * + * 2. Class Definitions: + * - `generateClassDefOfType` creates class definitions by randomly generating attributes and method declarations. + * + * 3. Declarations: + * - Variable and function declarations are produced, with variable types and function signatures adhering to + * the current scope and type constraints. + * - Generates variable definitions, type annotations, and initial assignments. + * + * 4. Statements and Blocks: + * - Statements include simple and compound types like assignments, expressions, control flows (`if`, `while`, + * `for`), and function calls. + * - Blocks of statements and declarations are recursively generated, maintaining indentation and scope context. + * + * Expression Generation: + * + * 1. Expressions: + * - Consist of literals, variable references, calls, binary/unary operations, and more. + * - Generated expressions match the expected type and respect the nesting depth to prevent excessive recursion. + * + * 2. Expression Types: + * - Handles special types (int, bool, str), objects, lists, and other types (None and Empty). + * - Ensures that generated expressions conform to the expected type through type-driven choices. + * + * Auxiliary Functions: + * + * 1. Utility Functions: + * - Functions like `generateIdentifier`, `generateLiteralOfType`, and others support generating syntactically + * correct identifiers and literals. + * - `generateConformingType` generates types selected conform to the required constraints and hierarchies. + * + * 2. Random Bounds and Lists: + * - Uses custom bounded random functions to select sizes and contents within specified maximums. + * - Generates lists of items for program components like statements, attributes, etc., using defined constraints. + * + * Recursive Depth Management: + * + * 1. Depth Management: + * - Manages recursion and nesting depth to prevent infinite loops and overly complex constructs. + * - Controls branching choices to balance between simple and compound constructs based on current depth and randomness. + * + * Scoping and Context: + * + * 1. Scope Maintenance: + * - Tracks variable declarations, types, and function scopes to ensure valid references. + * - Manages global and local scopes, ensuring access to variables and functions in the correct context. + * + */ + public class ChocoPySemanticGeneratorTypeDirected extends Generator { public ChocoPySemanticGeneratorTypeDirected() { super(String.class); // Register type of generated object From e94f0570dad7cc5c32d806860beadb190456a761 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 19 Sep 2024 16:29:16 -0400 Subject: [PATCH 13/14] Added type directed targets --- .../chocopy/SemanticAnalysisTest.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java index 43dcd4a69..ea9418ae5 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java @@ -10,8 +10,6 @@ import edu.berkeley.cs.jqf.fuzz.JQF; import org.junit.runner.RunWith; -import static edu.berkeley.cs.jqf.fuzz.util.Observability.event; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @RunWith(JQF.class) @@ -19,14 +17,20 @@ public class SemanticAnalysisTest { /** Entry point for fuzzing reference ChocoPy semantic analysis with ChocoPy code generator */ @Fuzz - public void fuzzSemanticAnalysis(@From(ChocoPySemanticGeneratorTypeDirected.class) String code) { + public void fuzzSemanticAnalysis(@From(ChocoPySemanticGenerator.class) String code) { Program program = RefParser.process(code, false); assumeTrue(!program.hasErrors()); - event("numStatements", program.statements.size()); - event("numDeclarations", program.declarations.size()); Program typedProgram = RefAnalysis.process(program); - event("numErrors", program.getErrorList().size()); - assertTrue(!typedProgram.hasErrors()); + assumeTrue(!typedProgram.hasErrors()); + } + + /** Entry point for fuzzing reference ChocoPy semantic analysis with ChocoPy code generator */ + @Fuzz + public void fuzzSemanticAnalysisTypeDirected(@From(ChocoPySemanticGeneratorTypeDirected.class) String code) { + Program program = RefParser.process(code, false); + assumeTrue(!program.hasErrors()); + Program typedProgram = RefAnalysis.process(program); + assumeTrue(!typedProgram.hasErrors()); } @Fuzz @@ -37,4 +41,13 @@ public void fuzzCodeGen(@From(ChocoPySemanticGenerator.class) String code) { assumeTrue(!typedProgram.hasErrors()); RefCodeGen.process(typedProgram); } + + @Fuzz + public void fuzzCodeGenTypeDirected(@From(ChocoPySemanticGeneratorTypeDirected.class) String code) { + Program program = RefParser.process(code, false); + assumeTrue(!program.hasErrors()); + Program typedProgram = RefAnalysis.process(program); + assumeTrue(!typedProgram.hasErrors()); + RefCodeGen.process(typedProgram); + } } From f4d6d08b1c85b4ef7c1570445bfa2e359eb08ab6 Mon Sep 17 00:00:00 2001 From: Vasu Vikram Date: Thu, 19 Sep 2024 16:33:28 -0400 Subject: [PATCH 14/14] Rename test --- .../chocopy/{SemanticAnalysisTest.java => ChocoPyTest.java} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/{SemanticAnalysisTest.java => ChocoPyTest.java} (97%) diff --git a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPyTest.java similarity index 97% rename from examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java rename to examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPyTest.java index ea9418ae5..d83112e78 100644 --- a/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/SemanticAnalysisTest.java +++ b/examples/src/test/java/edu/berkeley/cs/jqf/examples/chocopy/ChocoPyTest.java @@ -1,6 +1,5 @@ package edu.berkeley.cs.jqf.examples.chocopy; -import chocopy.ChocoPy; import chocopy.common.astnodes.Program; import chocopy.reference.RefAnalysis; import chocopy.reference.RefCodeGen; @@ -13,7 +12,7 @@ import static org.junit.Assume.assumeTrue; @RunWith(JQF.class) -public class SemanticAnalysisTest { +public class ChocoPyTest { /** Entry point for fuzzing reference ChocoPy semantic analysis with ChocoPy code generator */ @Fuzz