Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tyche logging support #258

Merged
merged 4 commits into from
Sep 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions fuzz/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@
<artifactId>eclipse-collections</artifactId>
<version>10.4.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.2</version>
<scope>compile</scope>
</dependency>
</dependencies>


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,11 @@ public void finalize() {
}
}

@Override
public String observeGuidance() {
return "AFL";
}


/**
* Returns an input stream containing the bytes that AFL
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,14 @@ protected void appendLineToFile(File file, String line) throws GuidanceException

}

@Override
public String observeGuidance() {
if (blind) {
return "Random";
}
return "Zest";
}

/* Writes a line of text to the log file. */
protected void infoLog(String str, Object... args) {
if (verbose) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,14 @@ default void observeGeneratedArgs(Object[] args) {
// Do nothing
}

/**
* <p>This method is only invoked during optional observability
* logging to provide the type of guidance used for input generation.</p>
*/
default String observeGuidance() {
return "";
}

/**
* Handles the end of a fuzzing trial.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,17 @@
import com.pholser.junit.quickcheck.internal.ParameterTypeContext;
import com.pholser.junit.quickcheck.internal.generator.GeneratorRepository;
import com.pholser.junit.quickcheck.random.SourceOfRandomness;
import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance;
import edu.berkeley.cs.jqf.fuzz.guidance.Guidance;
import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException;
import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException;
import edu.berkeley.cs.jqf.fuzz.guidance.Result;
import edu.berkeley.cs.jqf.fuzz.guidance.StreamBackedRandom;
import edu.berkeley.cs.jqf.fuzz.junit.TrialRunner;
import edu.berkeley.cs.jqf.fuzz.random.NoGuidance;
import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance;
import edu.berkeley.cs.jqf.instrument.InstrumentationException;
import edu.berkeley.cs.jqf.fuzz.util.Observability;
import org.junit.AssumptionViolatedException;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.MultipleFailureException;
Expand All @@ -73,6 +77,7 @@ public class FuzzStatement extends Statement {
private final List<Class<?>> expectedExceptions;
private final List<Throwable> failures = new ArrayList<>();
private final Guidance guidance;
private final Observability observability;
private boolean skipExceptionSwallow;

public FuzzStatement(FrameworkMethod method, TestClass testClass,
Expand All @@ -85,6 +90,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass,
this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes());
this.guidance = fuzzGuidance;
this.skipExceptionSwallow = Boolean.getBoolean("jqf.failOnDeclaredExceptions");
this.observability = new Observability(testClass.getName(), method.getName(), System.currentTimeMillis());
}

/**
Expand All @@ -101,16 +107,19 @@ public void evaluate() throws Throwable {
.collect(Collectors.toList());

// Keep fuzzing until no more input or I/O error with guidance
// Get current time in unix timestamp
long endGenerationTime = 0;
try {

// Keep fuzzing as long as guidance wants to
while (guidance.hasInput()) {
Result result = INVALID;
Throwable error = null;
long startTrialTime = System.currentTimeMillis();

// Initialize guided fuzzing using a file-backed random number source
Object [] args = {};
try {
Object[] args;
try {

// Generate input values
Expand Down Expand Up @@ -142,6 +151,7 @@ public void evaluate() throws Throwable {
throw new GuidanceException(e);
}

endGenerationTime = System.currentTimeMillis();
// Attempt to run the trial
guidance.run(testClass, method, args);

Expand Down Expand Up @@ -170,6 +180,17 @@ public void evaluate() throws Throwable {
failures.add(e);
}
}
long endTrialTime = System.currentTimeMillis();
if (System.getProperty("jqfObservability") != null) {
vasumv marked this conversation as resolved.
Show resolved Hide resolved
observability.addStatus(result);
if (result == SUCCESS) {
observability.addTiming(startTrialTime, endGenerationTime, endTrialTime);
}
observability.addArgs(args);
observability.add("how_generated", guidance.observeGuidance());

observability.writeToFile();
}

// Inform guidance about the outcome of this trial
try {
Expand All @@ -180,8 +201,6 @@ public void evaluate() throws Throwable {
// Anything else thrown from handleResult is an internal error, so wrap
throw new GuidanceException(e);
}


}
} catch (GuidanceException e) {
System.err.println("Fuzzing stopped due to guidance exception: " + e.getMessage());
Expand All @@ -198,6 +217,7 @@ public void evaluate() throws Throwable {
}
}


}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,11 @@ public void observeGeneratedArgs(Object[] args) {
}
}

@Override
public String observeGuidance() {
return "Repro";
}

/**
* Writes an object to a file
*
Expand Down
110 changes: 110 additions & 0 deletions fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package edu.berkeley.cs.jqf.fuzz.util;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import edu.berkeley.cs.jqf.fuzz.guidance.Result;

import java.io.FileWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;

import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE;
import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID;

public class Observability {

private final ObjectMapper objectMapper = new ObjectMapper();
private final String testClass;
private final String testMethod;
private final Path obsPath;
private static ObjectNode testCaseJsonObject;
private final long startTime;

public Observability(String testClass, String testMethod, long startTime) {
this.testClass = testClass;
this.testMethod = testMethod;
this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl");
if (obsPath.toFile().exists()) {
obsPath.toFile().delete();
}
this.startTime = startTime;
this.initializeTestCase();
}

public void initializeTestCase() {
testCaseJsonObject = objectMapper.createObjectNode();
testCaseJsonObject.putObject("features");
testCaseJsonObject.putObject("timing");
testCaseJsonObject.putObject("coverage");
testCaseJsonObject.putObject("args");
testCaseJsonObject.putObject("metadata");
testCaseJsonObject.put("type", "test_case");
testCaseJsonObject.put("run_start", startTime);
testCaseJsonObject.put("property", testMethod);
}

public static void event(String value, Object payload) throws RuntimeException {
// Add the payload to the features object
JsonNode jsonFeaturesNode = testCaseJsonObject.get("features");
ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode;

if (payload instanceof Integer) {
featuresNode.put(value, (Integer) payload);
} else if (payload instanceof String) {
featuresNode.put(value, (String) payload);
} else if (payload instanceof Float) {
featuresNode.put(value, (Float) payload);
} else {
throw new RuntimeException("Unsupported payload type for event");
}
}

public void addStatus(Result result) {
if (result == INVALID) {
testCaseJsonObject.put("status", "gave_up");
testCaseJsonObject.put("status_reason", "assumption violated");
} else if (result == FAILURE) {
testCaseJsonObject.put("status", "failed");
testCaseJsonObject.put("status_reason", "Encountered exception");
} else {
testCaseJsonObject.put("status", "passed");
testCaseJsonObject.put("status_reason", "");
}

}

public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) {
JsonNode timingNode = testCaseJsonObject.get("timing");
ObjectNode timingObject = (ObjectNode) timingNode;
timingObject.put("generation", endGenerationTime - startTime);
timingObject.put("execution", endExecutionTime - endGenerationTime);
}

public void addArgs(Object[] args) {
JsonNode argsNode = testCaseJsonObject.get("args");
ObjectNode argsObject = (ObjectNode) argsNode;
for (int i = 0; i < args.length; i++) {
argsObject.put("arg" + i, args[i].toString());
}
add("representation", Arrays.toString(args));
}

public void add(String key, String value) {
testCaseJsonObject.put(key, value);
}

public void writeToFile() {
// Append the JSON object to a file followed by a newline
try {
String jsonString = objectMapper.writeValueAsString(testCaseJsonObject);
try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) {
writer.write(jsonString);
writer.write(System.lineSeparator()); // Add a new line after each object
}
} catch (Exception e) {
throw new RuntimeException("Failed to write observations to file", e);
}
}
}
Loading