Skip to content

Commit 97985fa

Browse files
fwhdzhfwhdzh
committed
minor update.
1 parent 95f48e8 commit 97985fa

File tree

19 files changed

+456
-173
lines changed

19 files changed

+456
-173
lines changed

‎README.md

Lines changed: 91 additions & 88 deletions
Large diffs are not rendered by default.

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/CloudFuzzMain.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ public static void main(String[] args) throws IOException {
2727
Conf conf = new Conf(confFile);
2828
// conf.CONTROLLER_PORT = Integer.parseInt(args[0].trim());
2929
conf.loadConfigurationAndCheckAndPrint();
30+
31+
if ((args.length == 2) && (args[1].equals("recover"))) {
32+
conf.RECOVERY_MODE = true;
33+
}
3034

3135
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
3236
//System.out.println(runtimeMXBean.getName());

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/Conf.java

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,6 @@ public class Conf {
6060
public String ROOTDIR;
6161

6262
public boolean RECOVERY_MODE = false;
63-
// public String RECOVERY_ROOT_PATH = "/data/fengwenhan/data/crashfuzz_fwh";
64-
// public String RECOVERY_FUZZINFO_PATH = "/data/fengwenhan/data/crashfuzz_fwh/FuzzInfo.txt";
65-
// public String RECOVERY_CANDIDATEQUEUE_PATH = "/data/fengwenhan/data/crashfuzz_fwh/CandidateQueue.txt";
66-
// public String RECOVERY_TESTEDFAULTID_PATH = "/data/fengwenhan/data/crashfuzz_fwh/TestedFaultId.txt";
67-
// public String RECOVERY_VIRGINBITS_PATH = "/data/fengwenhan/data/crashfuzz_fwh/VirginBits.txt";
6863
public String RECOVERY_FUZZINFO_PATH;
6964
public String RECOVERY_CANDIDATEQUEUE_PATH;
7065
public String RECOVERY_TESTEDFAULTID_PATH;
@@ -280,10 +275,10 @@ public void loadConfiguration() throws IOException {
280275
if(!recoveryDir.startsWith("/")) {
281276
recoveryDir = workdir + recoveryDir;
282277
}
283-
RECOVERY_FUZZINFO_PATH = recoveryDir + "/FuzzInfo.txt";
284-
RECOVERY_CANDIDATEQUEUE_PATH = recoveryDir + "/CandidateQueue.txt";
285-
RECOVERY_TESTEDFAULTID_PATH = recoveryDir + "/TestedFaultId.txt";
286-
RECOVERY_VIRGINBITS_PATH = recoveryDir + "/VirginBits.txt";
278+
RECOVERY_FUZZINFO_PATH = recoveryDir + "/" + JSONBasedRecoveryManager.candidateQueueFile;
279+
RECOVERY_CANDIDATEQUEUE_PATH = recoveryDir + "/" + JSONBasedRecoveryManager.candidateQueueFile;
280+
RECOVERY_TESTEDFAULTID_PATH = recoveryDir + "/" + JSONBasedRecoveryManager.testedFaultIdFile;
281+
RECOVERY_VIRGINBITS_PATH = recoveryDir + "/" + JSONBasedRecoveryManager.virginBitsFile;
287282
}
288283

289284
String determineWaitTime = p.getProperty(ConfOption.DETERMINE_WAIT_TIME.toString());

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/FaultFuzzRecovery.java

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import java.util.HashMap;
88
import java.util.List;
99
import java.util.Map;
10+
import java.util.regex.Matcher;
11+
import java.util.regex.Pattern;
1012

1113
import com.alibaba.fastjson.JSON;
1214
import com.alibaba.fastjson.JSONObject;
@@ -37,7 +39,6 @@ public void recoverFuzzInfo(String filepath) {
3739
}
3840

3941
public void recoverCandidateQueue(String rootPath) throws IOException {
40-
4142
File root = new File(rootPath);
4243
File[] files = root.listFiles();
4344
//get "persist" subfolder
@@ -52,8 +53,6 @@ public void recoverCandidateQueue(String rootPath) throws IOException {
5253
for (File file : persistTestFiles) {
5354
System.out.println(file.getName());
5455
}
55-
56-
5756

5857
Map<String, QueueEntry> queueEntryMap = new HashMap<>();
5958
for (File file : persistTestFiles) {
@@ -103,9 +102,53 @@ public void recoverCandidateQueue(String rootPath) throws IOException {
103102
}
104103
}
105104

105+
for (String entryId : queueEntryMap.keySet()) {
106+
QueueEntry entry = queueEntryMap.get(entryId);
107+
File entryFolder = new File(persistFolder.getAbsolutePath() + "/" + entryId);
108+
File[] entryFiles = entryFolder.listFiles();
109+
// File mapFile = null;
110+
for (File file : entryFiles) {
111+
if (file.getName().startsWith("MAP_")) {
112+
// mapFile = file;
113+
int bitmapSize = getBitmapSizeFromMapFileName(file.getName());
114+
if (bitmapSize >= 0) {
115+
entry.bitmap_size = bitmapSize;
116+
}
117+
break;
118+
}
119+
}
120+
}
121+
122+
for (String entryId : queueEntryMap.keySet()) {
123+
QueueEntry entry = queueEntryMap.get(entryId);
124+
File execTimeFile = new File(persistFolder.getAbsolutePath() + "/" + entryId + "/" + FileUtil.exec_second_file);
125+
if (execTimeFile.exists()) {
126+
List<String> lines = Files.readAllLines(execTimeFile.toPath());
127+
if (lines.size() > 0) {
128+
String execTimeStr = lines.get(0);
129+
long execSeconds = FileUtil.parseStringTimeToSeconds(execTimeStr);
130+
entry.exec_s = execSeconds;
131+
}
132+
}
133+
}
134+
106135
System.out.println("candidateQueue size: " + candidateQueue.size());
107136
}
108137

138+
public static int getBitmapSizeFromMapFileName(String mapFileName) {
139+
Pattern pattern = Pattern.compile("MAP_(\\d+)\\(\\d+\\)");
140+
Matcher matcher = pattern.matcher(mapFileName);
141+
if (matcher.find()) {
142+
String number = matcher.group(1);
143+
// System.out.println("Extracted Number: " + number);
144+
int bitmapSize = Integer.parseInt(number);
145+
return bitmapSize;
146+
} else {
147+
System.out.println("No number found in the string.");
148+
return -1;
149+
}
150+
}
151+
109152
public void recordVirginBits(String filepath) {
110153
CoverageCollector.write_bitmap(CoverageCollector.virgin_bits, filepath);
111154
}

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/Fuzzer.java

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,12 @@ public void run() {
582582
FileUtil.clearRootPath();
583583
for (int i = 0; i < Conf.WORKLOADLIST.size(); i++) {
584584

585+
if (checkPause()) {
586+
Stat.log("FaultFuzzer is recording, it may take a long time ...");
587+
recoveryManager.recordAll(conf.ROOTDIR + "/" + JSONBasedRecoveryManager.jsonRecordFolder);
588+
Stat.log("FaultFuzzer recording process has finished");
589+
}
590+
585591
while (checkPause()) {
586592
try {
587593
Thread.sleep(1000);
@@ -595,7 +601,29 @@ public void run() {
595601
performNoFaultRun(i);
596602
}
597603
} else {
598-
recovery();
604+
// recovery();
605+
Stat.log("FaultFuzzer is recovering, it may take a long time ...");
606+
recoveryManager.recoverAll(conf.ROOTDIR + "/" + JSONBasedRecoveryManager.jsonRecordFolder);
607+
Stat.log("FaultFuzzer recovering process has finished");
608+
if (FuzzInfo.total_execs < Conf.WORKLOADLIST.size()) {
609+
for (int i = (int) FuzzInfo.total_execs; i < Conf.WORKLOADLIST.size(); i++) {
610+
if (checkPause()) {
611+
Stat.log("FaultFuzzer is recording, it may take a long time ...");
612+
recoveryManager.recordAll(conf.ROOTDIR + "/" + JSONBasedRecoveryManager.jsonRecordFolder);
613+
Stat.log("FaultFuzzer recording process has finished");
614+
}
615+
while (checkPause()) {
616+
try {
617+
Thread.sleep(1000);
618+
FuzzInfo.pauseSecond++;
619+
Stat.log("FaultFuzzer is paused, waiting for resume...");
620+
} catch (InterruptedException e) {
621+
e.printStackTrace();
622+
}
623+
}
624+
performNoFaultRun(i);
625+
}
626+
}
599627
}
600628
performOtherRun();
601629

@@ -604,6 +632,8 @@ public void run() {
604632

605633
}
606634

635+
boolean hasRecorded = false;
636+
607637
public boolean checkPause() {
608638
boolean pause = false;
609639
File pauseFile = new File(FileUtil.root + FileUtil.pause_file);
@@ -617,13 +647,13 @@ public boolean checkPause() {
617647

618648
public void recovery() {
619649
Stat.log("recoveryFuzzInfo...");
620-
recoveryManager.recoverFuzzInfo(this);
650+
recoveryManager.recoverFuzzInfo(conf.RECOVERY_FUZZINFO_PATH);
621651
Stat.log("recoveryCandidateQueue...");
622-
recoveryManager.recoverCandidateQueue(this);
652+
recoveryManager.recoverCandidateQueue(conf.RECOVERY_CANDIDATEQUEUE_PATH);
623653
Stat.log("recoveryTestedFaultId...");
624-
recoveryManager.recoverTestedFaultId(this);
654+
recoveryManager.recoverTestedFaultId(conf.RECOVERY_TESTEDFAULTID_PATH);
625655
Stat.log("recoveryVirginBits...");
626-
recoveryManager.recoverVirginBits(this);
656+
recoveryManager.recoverVirginBits(conf.RECOVERY_VIRGINBITS_PATH);
627657
}
628658

629659
private void updateFavorListOfPairSeed(SelectionInfo.QueuePair q, FaultPoint injected_fault) {

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/JSONBasedRecoveryManager.java

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public void recoverCandidateQueue(String filepath) {
4141
oriList = Files.readAllLines(file.toPath());
4242
String s = oriList.get(0);
4343
List<QueueEntry> c = JSON.parseArray(s, QueueEntry.class);
44-
Stat.log(JSONObject.toJSONString(c));
44+
// Stat.log(JSONObject.toJSONString(c));
4545
Fuzzer.candidate_queue = c;
4646
} catch (IOException e) {
4747
e.printStackTrace();
@@ -116,7 +116,7 @@ public void recoverTestedFaultId(String filepath) {
116116
oriList = Files.readAllLines(file.toPath());
117117
String s = oriList.get(0);
118118
Set<Integer> c = (Set)JSON.parseObject(s, Set.class);
119-
Stat.log(JSONObject.toJSONString(c));
119+
// Stat.log(JSONObject.toJSONString(c));
120120
SelectionInfo.tested_fault_id = c;
121121
} catch (IOException e) {
122122
e.printStackTrace();
@@ -162,4 +162,37 @@ public void recoverVirginBits(Fuzzer fuzzer) {
162162
recoverVirginBits(fuzzer.conf.RECOVERY_VIRGINBITS_PATH);
163163
}
164164

165+
public void recordAll(String recoveryRootPath) {
166+
File dir = new File(recoveryRootPath);
167+
if (!dir.exists()) {
168+
dir.mkdirs();
169+
}
170+
recordCandidateQueue(recoveryRootPath + "/" + candidateQueueFile);
171+
recordFuzzInfo(recoveryRootPath + "/" + fuzzInfoFile);
172+
recordTestedFaultId(recoveryRootPath + "/" + testedFaultIdFile);
173+
recordVirginBits(recoveryRootPath + "/" + virginBitsFile);
174+
}
175+
176+
public void recoverAll(String recoveryRootPath) {
177+
if (Fuzzer.candidate_queue == null || Fuzzer.candidate_queue.size() == 0) {
178+
recoverCandidateQueue(recoveryRootPath + "/" + candidateQueueFile);
179+
}
180+
if (FuzzInfo.exec_us == 0) {
181+
recoverFuzzInfo(recoveryRootPath + "/" + fuzzInfoFile);
182+
}
183+
if (SelectionInfo.tested_fault_id == null || SelectionInfo.tested_fault_id.size() == 0) {
184+
recoverTestedFaultId(recoveryRootPath + "/" + testedFaultIdFile);
185+
}
186+
if (CoverageCollector.virgin_bits == null || CoverageCollector.virgin_bits.length == 0) {
187+
recoverVirginBits(recoveryRootPath + "/" + virginBitsFile);
188+
}
189+
}
190+
191+
public final static String jsonRecordFolder = "jsonRecord";
192+
public final static String candidateQueueFile = "candidateQueue.txt";
193+
public final static String fuzzInfoFile = "fuzzInfo.txt";
194+
public final static String testedFaultIdFile = "testedFaultId.txt";
195+
public final static String virginBitsFile = "virginBits.txt";
196+
197+
165198
}

‎ctrl/src/main/java/edu/iscas/tcse/faultfuzz/ctrl/utils/FileUtil.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public class FileUtil {
4646

4747
public final static String seed_file = "SEED";
4848

49-
public final static String fuzzed_time_file = "FUZZED_TIME";
49+
// public final static String fuzzed_time_file = "FUZZED_TIME";
5050
public final static String mutates_size_file = "MUTATES_SIZE";
5151
public final static String handicap_file = "HANDICAP";
5252
public final static String mutates_file = "MUTATES";

‎ctrl/src/test/java/edu/iscas/tcse/faultfuzz/ctrl/FaultFuzzRecoveryTest.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,33 @@
11
package edu.iscas.tcse.faultfuzz.ctrl;
22

33
import java.io.IOException;
4+
import java.util.regex.Matcher;
5+
import java.util.regex.Pattern;
46

57
import org.junit.Test;
68

79
public class FaultFuzzRecoveryTest {
810

11+
@Test
12+
public void test() {
13+
String input = "MAP_1785(130)";
14+
Pattern pattern = Pattern.compile("MAP_(\\d+)\\(\\d+\\)");
15+
Matcher matcher = pattern.matcher(input);
16+
if (matcher.find()) {
17+
String number = matcher.group(1);
18+
System.out.println("Extracted Number: " + number);
19+
} else {
20+
System.out.println("No number found in the string.");
21+
}
22+
}
23+
924
@Test
1025
public void testRecoverCandidateQueue() throws IOException {;
1126
// Conf conf = Mockito.mock(Conf.class);
1227
// conf.MAX_FAULTS = 10;
1328
// List<MaxDownNodes> maxDownGroup = conf.parseMaxDownGroup("2:{172.30.0.2,172.30.0.3,172.30.0.4,172.30.0.5,172.30.0.6}");
1429
// conf.maxDownGroup = maxDownGroup;
30+
// conf.CUR_FAULT_FILE = new File("faultUnderTest");
1531
// FaultFuzzRecovery recover = new FaultFuzzRecovery(conf);
1632
// recover.recoverCandidateQueue("/data/fengwenhan/data/faultfuzz_recovery_test");
1733
}

‎ctrl/src/test/java/edu/iscas/tcse/faultfuzz/ctrl/JSONBasedRecoveryManagerTest.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package edu.iscas.tcse.faultfuzz.ctrl;
2+
3+
import java.io.IOException;
4+
5+
import org.junit.Test;
6+
7+
public class JSONBasedRecoveryManagerTest {
8+
@Test
9+
public void testRecoverAll() throws IOException {
10+
// String confPath = "/data/fengwenhan/code/faultfuzz/package/zk-3.6.3/backend-configuration/FaultFuzz-backend-configuration.properties";
11+
// String args[] = new String[2];
12+
// args[0] = confPath;
13+
// args[1] = "recover";
14+
// CloudFuzzMain.main(args);
15+
16+
}
17+
}

‎doc/Configuration.md

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,35 @@
1-
The configuration options supported by FaultFuzz-ctrl include:
1+
Note that the configurations in the
2+
configuration file are more low-level and slightly different from the configuration items
3+
provided in our FaultFuzz frontend website. In the FaultFuzz frontend website, we have
4+
simplified the configuration information to make it more human-understandable.
5+
6+
There are two configuration files used in FaultFuzz, i.e., configuration file for FaultFuzz backend
7+
(named FaultFuzz-backend-configuration.properties if you generate it by FaultFuzz frontend) and configuration file
8+
for FaultFuzz observer and the system under test (named FaultFuzz-SUT-configuration.sh if you generate it by FaultFuzz frontend).
9+
10+
FaultFuzz-backend-configuration.properties is property file, which looks like:
11+
```
12+
WORKLOAD={/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/workload-1.sh,/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/workload-2.sh}
13+
CHECKER=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/detectFailureSymptoms.sh
14+
FAULT_TYPE=[CRASH,REBOOT,NETWORK_DISCONNECTION,NETWORK_RECONNECTION]
15+
CRASH=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/crashNode.sh
16+
REBOOT=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/startNode.sh
17+
NETWORK_DISCONNECTION=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/network-disconnect.sh
18+
NETWORK_RECONNECTION=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/network-connect.sh
19+
ROOT_DIR=/data/faultfuzz_zk
20+
CUR_FAULT_FILE=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/faultUnderTest
21+
CONTROLLER_PORT=12090
22+
MONITOR=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/copyRuntimeInformation.sh
23+
PRETREATMENT=/zookeeper/faultfuzz/package/zk-3.6.3/backend-configuration/resetSystem.sh
24+
TEST_TIME=80h
25+
FAULT_CSTR=2:{172.30.0.2,172.30.0.3,172.30.0.4,172.30.0.5,172.30.0.6}
26+
AFL_PORT=12081
27+
HANG_TIMEOUT=10m
28+
MAX_FAULTS=10
29+
DETERMINE_WAIT_TIME=30000
30+
```
31+
32+
The configuration options supported by FaultFuzz-backend-configuration.properties include:
233

334
- **WORKLOAD**: The string path for the script used for running a workload.
435
- **CHECKER**: The string path for the script used for checking failure symptoms.
@@ -43,7 +74,23 @@ The configuration options supported by FaultFuzz-ctrl include:
4374
giving up to wait next I/O operation.
4475

4576

46-
The configuration options supported by FaultFuzz-inst include:
77+
FaultFuzz-SUT-configuration.sh is a linux bash file. It provides two Linux environment variables, i.e., `FAV_OPTS` and `PHOS_OPTS`.
78+
Where FAV_OPTS is an environment variable generated based on the user-defined
79+
configuration. PHOS_OPTS, on the other hand, retains additional information
80+
introduced by FaultFuzz on top of FAV_OPTS, but without control. PHOS_OPTS is
81+
particularly useful in distributed systems where multiple processes interact.
82+
For instance, in the case of Zookeeper, we may want the server processes to be
83+
controlled by FaultFuzz while the client processes remain uncontrolled.
84+
85+
An example of FaultFuzz-SUT-configuration.sh looks like:
86+
87+
```
88+
export PHOS_OPTS="-Xbootclasspath/a:/SUT-configuration/FaultFuzz-inst-0.0.5-SNAPSHOT.jar -javaagent:/SUT-configuration/FaultFuzz-inst-0.0.5-SNAPSHOT.jar=useFaultFuzz=false"
89+
90+
export FAV_OPTS="-Xbootclasspath/a:/SUT-configuration/FaultFuzz-inst-0.0.5-SNAPSHOT.jar -javaagent:/SUT-configuration/FaultFuzz-inst-0.0.5-SNAPSHOT.jar=useFaultFuzz=true,forZk=true,jdkFile=true,observerHome=/observer,dataPaths=/zookeeper-3.6.3/zkData/version-2,controllerSocket=172.30.0.1:12090,covIncludes=org/apache/zookeeper,aflAllow=/SUT-configuration/allowlist,aflDeny=/SUT-configuration/denylist,aflPort=12081"
91+
```
92+
93+
The configuration options supported by FaultFuzz-SUT-configuration.sh include:
4794

4895
- **useFaultFuzz**: Controls the usage of FaultFuzz for the program or process.
4996

0 commit comments

Comments
 (0)