Tyche logging support

fuzz/pom.xml

@@ -55,6 +55,12 @@
  <artifactId>eclipse-collections</artifactId>
  <version>10.4.0</version>
  </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.15.2</version>
+ <scope>compile</scope>
+ </dependency>
  </dependencies>
 
 

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java

@@ -41,13 +41,17 @@
 import com.pholser.junit.quickcheck.internal.ParameterTypeContext;
 import com.pholser.junit.quickcheck.internal.generator.GeneratorRepository;
 import com.pholser.junit.quickcheck.random.SourceOfRandomness;
+import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance;
 import edu.berkeley.cs.jqf.fuzz.guidance.Guidance;
 import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException;
 import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException;
 import edu.berkeley.cs.jqf.fuzz.guidance.Result;
 import edu.berkeley.cs.jqf.fuzz.guidance.StreamBackedRandom;
 import edu.berkeley.cs.jqf.fuzz.junit.TrialRunner;
+import edu.berkeley.cs.jqf.fuzz.random.NoGuidance;
+import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance;
 import edu.berkeley.cs.jqf.instrument.InstrumentationException;
+import edu.berkeley.cs.jqf.fuzz.util.Observability;
 import org.junit.AssumptionViolatedException;
 import org.junit.runners.model.FrameworkMethod;
 import org.junit.runners.model.MultipleFailureException;
@@ -73,6 +77,7 @@ public class FuzzStatement extends Statement {
  private final List<Class<?>> expectedExceptions;
  private final List<Throwable> failures = new ArrayList<>();
  private final Guidance guidance;
+ private final Observability observability;
  private boolean skipExceptionSwallow;
 
  public FuzzStatement(FrameworkMethod method, TestClass testClass,
@@ -85,6 +90,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass,
  this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes());
  this.guidance = fuzzGuidance;
  this.skipExceptionSwallow = Boolean.getBoolean("jqf.failOnDeclaredExceptions");
+ this.observability = new Observability(testClass.getName(), method.getName(), System.currentTimeMillis());
  }
 
  /**
@@ -101,16 +107,19 @@ public void evaluate() throws Throwable {
  .collect(Collectors.toList());
 
  // Keep fuzzing until no more input or I/O error with guidance
+ // Get current time in unix timestamp
+ long endGenerationTime = 0;
  try {
 
  // Keep fuzzing as long as guidance wants to
  while (guidance.hasInput()) {
  Result result = INVALID;
  Throwable error = null;
+ long startTrialTime = System.currentTimeMillis();
 
  // Initialize guided fuzzing using a file-backed random number source
+ Object [] args = {};
  try {
- Object[] args;
  try {
 
  // Generate input values
@@ -142,6 +151,7 @@ public void evaluate() throws Throwable {
  throw new GuidanceException(e);
  }
 
+ endGenerationTime = System.currentTimeMillis();
  // Attempt to run the trial
  guidance.run(testClass, method, args);
 
@@ -170,6 +180,26 @@ public void evaluate() throws Throwable {
  failures.add(e);
  }
  }
+ long endTrialTime = System.currentTimeMillis();
+ if (System.getProperty("jqfObservability") != null) {
+ observability.addStatus(result);
+ if (result == SUCCESS) {
+ observability.addTiming(startTrialTime, endGenerationTime, endTrialTime);
+ }
+ observability.addArgs(args);
+
+ if (guidance instanceof ZestGuidance) {
+ observability.add("how_generated", "Zest");
+ } else if (guidance instanceof NoGuidance) {
+ observability.add("how_generated", "random");
+ } else if (guidance instanceof ReproGuidance) {
+ observability.add("how_generated", "repro");
+ } else {
+ observability.add("how_generated", "unknown");
+ }
+
+ observability.writeToFile();
+ }
 
  // Inform guidance about the outcome of this trial
  try {
@@ -180,8 +210,6 @@ public void evaluate() throws Throwable {
  // Anything else thrown from handleResult is an internal error, so wrap
  throw new GuidanceException(e);
  }
-
-
  }
  } catch (GuidanceException e) {
  System.err.println("Fuzzing stopped due to guidance exception: " + e.getMessage());
@@ -198,6 +226,7 @@ public void evaluate() throws Throwable {
  }
  }
 
+
  }
 
  /**

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java

@@ -0,0 +1,110 @@
+package edu.berkeley.cs.jqf.fuzz.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import edu.berkeley.cs.jqf.fuzz.guidance.Result;
+
+import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE;
+import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID;
+
+public class Observability {
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private final String testClass;
+ private final String testMethod;
+ private final Path obsPath;
+ private static ObjectNode testCaseJsonObject;
+ private final long startTime;
+
+ public Observability(String testClass, String testMethod, long startTime) {
+ this.testClass = testClass;
+ this.testMethod = testMethod;
+ this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl");
+ if (obsPath.toFile().exists()) {
+ obsPath.toFile().delete();
+ }
+ this.startTime = startTime;
+ this.initializeTestCase();
+ }
+
+ public void initializeTestCase() {
+ testCaseJsonObject = objectMapper.createObjectNode();
+ testCaseJsonObject.putObject("features");
+ testCaseJsonObject.putObject("timing");
+ testCaseJsonObject.putObject("coverage");
+ testCaseJsonObject.putObject("args");
+ testCaseJsonObject.putObject("metadata");
+ testCaseJsonObject.put("type", "test_case");
+ testCaseJsonObject.put("run_start", startTime);
+ testCaseJsonObject.put("property", testMethod);
+ }
+
+ public static void event(String value, Object payload) throws RuntimeException {
+ // Add the payload to the features object
+ JsonNode jsonFeaturesNode = testCaseJsonObject.get("features");
+ ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode;
+
+ if (payload instanceof Integer) {
+ featuresNode.put(value, (Integer) payload);
+ } else if (payload instanceof String) {
+ featuresNode.put(value, (String) payload);
+ } else if (payload instanceof Float) {
+ featuresNode.put(value, (Float) payload);
+ } else {
+ throw new RuntimeException("Unsupported payload type for event");
+ }
+ }
+
+ public void addStatus(Result result) {
+ if (result == INVALID) {
+ testCaseJsonObject.put("status", "gave_up");
+ testCaseJsonObject.put("status_reason", "assumption violated");
+ } else if (result == FAILURE) {
+ testCaseJsonObject.put("status", "failed");
+ testCaseJsonObject.put("status_reason", "Encountered exception");
+ } else {
+ testCaseJsonObject.put("status", "passed");
+ testCaseJsonObject.put("status_reason", "");
+ }
+
+ }
+
+ public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) {
+ JsonNode timingNode = testCaseJsonObject.get("timing");
+ ObjectNode timingObject = (ObjectNode) timingNode;
+ timingObject.put("generation", endGenerationTime - startTime);
+ timingObject.put("execution", endExecutionTime - endGenerationTime);
+ }
+
+ public void addArgs(Object[] args) {
+ JsonNode argsNode = testCaseJsonObject.get("args");
+ ObjectNode argsObject = (ObjectNode) argsNode;
+ for (int i = 0; i < args.length; i++) {
+ argsObject.put("arg" + i, args[i].toString());
+ }
+ add("representation", Arrays.toString(args));
+ }
+
+ public void add(String key, String value) {
+ testCaseJsonObject.put(key, value);
+ }
+
+ public void writeToFile() {
+ // Append the JSON object to a file followed by a newline
+ try {
+ String jsonString = objectMapper.writeValueAsString(testCaseJsonObject);
+ try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) {
+ writer.write(jsonString);
+ writer.write(System.lineSeparator()); // Add a new line after each object
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to write observations to file", e);
+ }
+ }
+}