Skip to content

Commit b4028cc

Browse files
lu-pintolu-pinto
authored
Add missing ModExp test cases and improvements to evmtool benchmarks (#8757)
1 parent be260b5 commit b4028cc

File tree

12 files changed

+687
-303
lines changed

12 files changed

+687
-303
lines changed

‎ethereum/evmtool/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ dependencies {
5656
implementation 'com.google.guava:guava'
5757
implementation 'info.picocli:picocli'
5858
implementation 'io.vertx:vertx-core'
59+
implementation 'tools.profiler:async-profiler:4.0'
5960

6061
annotationProcessor 'com.google.dagger:dagger-compiler'
6162
annotationProcessor 'info.picocli:picocli-codegen'

‎ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@
1616

1717
import static org.hyperledger.besu.evmtool.BenchmarkSubCommand.COMMAND_NAME;
1818
import static picocli.CommandLine.ScopeType.INHERIT;
19+
import static picocli.CommandLine.ScopeType.LOCAL;
1920

2021
import org.hyperledger.besu.evm.precompile.AbstractBLS12PrecompiledContract;
2122
import org.hyperledger.besu.evm.precompile.AbstractPrecompiledContract;
2223
import org.hyperledger.besu.evmtool.benchmarks.AltBN128Benchmark;
2324
import org.hyperledger.besu.evmtool.benchmarks.BLS12Benchmark;
25+
import org.hyperledger.besu.evmtool.benchmarks.BenchmarkConfig;
2426
import org.hyperledger.besu.evmtool.benchmarks.BenchmarkExecutor;
2527
import org.hyperledger.besu.evmtool.benchmarks.ECRecoverBenchmark;
2628
import org.hyperledger.besu.evmtool.benchmarks.ModExpBenchmark;
@@ -30,6 +32,7 @@
3032

3133
import java.io.PrintStream;
3234
import java.util.EnumSet;
35+
import java.util.Optional;
3336

3437
import picocli.CommandLine;
3538
import picocli.CommandLine.Option;
@@ -52,21 +55,22 @@ public class BenchmarkSubCommand implements Runnable {
5255
*/
5356
public static final String COMMAND_NAME = "benchmark";
5457

58+
/** Stream for where to write the output to. */
5559
private final PrintStream output;
5660

5761
enum Benchmark {
58-
altBn128(new AltBN128Benchmark()),
62+
altBn128(AltBN128Benchmark::new),
5963
// blake2f
60-
EcRecover(new ECRecoverBenchmark()),
61-
ModExp(new ModExpBenchmark()),
62-
Secp256k1(new Secp256k1Benchmark()),
64+
EcRecover(ECRecoverBenchmark::new),
65+
ModExp(ModExpBenchmark::new),
66+
Secp256k1(Secp256k1Benchmark::new),
6367
// bls12
64-
Bls12(new BLS12Benchmark());
68+
Bls12(BLS12Benchmark::new);
6569

66-
final BenchmarkExecutor benchmarkExecutor;
70+
private final BenchmarkExecutor.Builder executorBuilder;
6771

68-
Benchmark(final BenchmarkExecutor benchmarkExecutor) {
69-
this.benchmarkExecutor = benchmarkExecutor;
72+
Benchmark(final BenchmarkExecutor.Builder executorBuilder) {
73+
this.executorBuilder = executorBuilder;
7074
}
7175
}
7276

@@ -75,7 +79,7 @@ enum Benchmark {
7579
description = "Use the native libraries.",
7680
scope = INHERIT,
7781
negatable = true)
78-
Boolean nativeCode;
82+
Boolean nativeCode = false;
7983

8084
@Option(
8185
names = {"--use-precompile-cache"},
@@ -84,6 +88,50 @@ enum Benchmark {
8488
negatable = true)
8589
Boolean enablePrecompileCache = false;
8690

91+
@Option(
92+
names = {"--async-profiler"},
93+
description =
94+
"Benchmark using async profiler. No profiler command means profiling disabled. '%%%%test-case' in the"
95+
+ " file name expands to the test for which the profiler ran,"
96+
+ "e.g. \"start,jfr,event=cpu,file=/tmp/%%%%test-case-%%p.jfr\".",
97+
scope = LOCAL)
98+
Optional<String> asyncProfilerOptions = Optional.empty();
99+
100+
@Option(
101+
names = {"--pattern"},
102+
description =
103+
"Only tests cases with this pattern will be run, e.g. --pattern \"guido-3.*\". Default runs all test cases.",
104+
scope = LOCAL)
105+
Optional<String> testCasePattern = Optional.empty();
106+
107+
@Option(
108+
names = {"--exec-iterations"},
109+
description =
110+
"Number of iterations that the benchmark should run (measurement) for, regardless of how long it takes.",
111+
scope = LOCAL)
112+
Optional<Integer> execIterations = Optional.empty();
113+
114+
@Option(
115+
names = {"--exec-time"},
116+
description =
117+
"Run the maximum number of iterations during execution (measurement) within the given period. Time is in seconds.",
118+
scope = LOCAL)
119+
Optional<Integer> execTime = Optional.empty();
120+
121+
@Option(
122+
names = {"--warm-iterations"},
123+
description =
124+
"Number of iterations that the benchmark should warm up for, regardless of how long it takes.",
125+
scope = LOCAL)
126+
Optional<Integer> warmIterations = Optional.empty();
127+
128+
@Option(
129+
names = {"--warm-time"},
130+
description =
131+
"Run the maximum number of iterations during warmup within the given period. Time is in seconds.",
132+
scope = LOCAL)
133+
Optional<Integer> warmTime = Optional.empty();
134+
87135
@Parameters(description = "One or more of ${COMPLETION-CANDIDATES}.")
88136
EnumSet<Benchmark> benchmarks = EnumSet.noneOf(Benchmark.class);
89137

@@ -107,13 +155,27 @@ public BenchmarkSubCommand(final PrintStream output) {
107155
@Override
108156
public void run() {
109157
LogConfigurator.setLevel("", "DEBUG");
110-
System.out.println(BesuVersionUtils.version());
158+
output.println(BesuVersionUtils.version());
111159
AbstractPrecompiledContract.setPrecompileCaching(enablePrecompileCache);
112160
AbstractBLS12PrecompiledContract.setPrecompileCaching(enablePrecompileCache);
113161
var benchmarksToRun = benchmarks.isEmpty() ? EnumSet.allOf(Benchmark.class) : benchmarks;
162+
final BenchmarkConfig benchmarkConfig =
163+
new BenchmarkConfig(
164+
nativeCode,
165+
enablePrecompileCache,
166+
asyncProfilerOptions,
167+
testCasePattern,
168+
execIterations,
169+
execTime,
170+
warmIterations,
171+
warmTime);
114172
for (var benchmark : benchmarksToRun) {
115-
System.out.println("Benchmarks for " + benchmark);
116-
benchmark.benchmarkExecutor.runBenchmark(output, nativeCode, parentCommand.getFork());
173+
output.println("Benchmarks for " + benchmark + " on fork " + parentCommand.getFork());
174+
BenchmarkExecutor executor = benchmark.executorBuilder.create(output, benchmarkConfig);
175+
if (executor.isPrecompile()) {
176+
BenchmarkExecutor.logPrecompileDerivedGasNotice(output);
177+
}
178+
executor.runBenchmark(nativeCode, parentCommand.getFork());
117179
}
118180
}
119181
}

‎ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/AltBN128Benchmark.java

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,18 @@
3030
/** Benchmark AltBN128 add, mul, and pairings */
3131
public class AltBN128Benchmark extends BenchmarkExecutor {
3232

33-
/** Benchmark AltBN128 add, mul, and pairings with default warmup and iterations */
34-
public AltBN128Benchmark() {
35-
super(MATH_WARMUP, MATH_ITERATIONS);
33+
/**
34+
* The constructor. Use default math based warmup and interations.
35+
*
36+
* @param output where to write the stats.
37+
* @param benchmarkConfig benchmark configurations.
38+
*/
39+
public AltBN128Benchmark(final PrintStream output, final BenchmarkConfig benchmarkConfig) {
40+
super(MATH_WARMUP, MATH_ITERATIONS, output, benchmarkConfig);
3641
}
3742

3843
@Override
39-
public void runBenchmark(
40-
final PrintStream output, final Boolean attemptNative, final String fork) {
44+
public void runBenchmark(final Boolean attemptNative, final String fork) {
4145

4246
EvmSpecVersion forkVersion = EvmSpecVersion.fromName(fork);
4347

@@ -71,12 +75,12 @@ private void benchmarkAdd(
7175
EvmSpecVersion.ISTANBUL.compareTo(forkVersion) < 0
7276
? AltBN128AddPrecompiledContract.byzantium(gasCalculator)
7377
: AltBN128AddPrecompiledContract.istanbul(gasCalculator);
74-
warmup = MATH_WARMUP / addTestCases.size();
75-
iterations = MATH_ITERATIONS / addTestCases.size();
78+
warmIterations = MATH_WARMUP / addTestCases.size();
79+
execIterations = MATH_ITERATIONS / addTestCases.size();
7680
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
7781
long gasCost = 0;
7882
for (final Map.Entry<String, Bytes> testCase : addTestCases.entrySet()) {
79-
execTime += runPrecompileBenchmark(testCase.getValue(), addContract);
83+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), addContract);
8084
gasCost += addContract.gasRequirement(testCase.getValue());
8185
}
8286
execTime /= addTestCases.size();
@@ -103,12 +107,12 @@ private void benchmarkMul(
103107
EvmSpecVersion.ISTANBUL.compareTo(forkVersion) < 0
104108
? AltBN128MulPrecompiledContract.byzantium(gasCalculator)
105109
: AltBN128MulPrecompiledContract.istanbul(gasCalculator);
106-
warmup = MATH_WARMUP / mulTestCases.size();
107-
iterations = MATH_ITERATIONS / mulTestCases.size();
110+
warmIterations = MATH_WARMUP / mulTestCases.size();
111+
execIterations = MATH_ITERATIONS / mulTestCases.size();
108112
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
109113
long gasCost = 0;
110114
for (final Map.Entry<String, Bytes> testCase : mulTestCases.entrySet()) {
111-
execTime += runPrecompileBenchmark(testCase.getValue(), mulContract);
115+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), mulContract);
112116
gasCost += mulContract.gasRequirement(testCase.getValue());
113117
}
114118
execTime /= mulTestCases.size();
@@ -168,11 +172,11 @@ private void benchmarkPairings(
168172
? AltBN128PairingPrecompiledContract.byzantium(gasCalculator)
169173
: AltBN128PairingPrecompiledContract.istanbul(gasCalculator);
170174

171-
warmup = MATH_WARMUP / 20;
172-
iterations = MATH_ITERATIONS / 20;
175+
warmIterations = MATH_WARMUP / 20;
176+
execIterations = MATH_ITERATIONS / 20;
173177

174178
for (int i = 0; i < pairings.length; i++) {
175-
final double execTime = runPrecompileBenchmark(pairings[i], contract);
179+
final double execTime = runPrecompileBenchmark("pairings" + i, pairings[i], contract);
176180
final long gasCost = contract.gasRequirement(pairings[i]);
177181

178182
output.printf(

‎ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BLS12Benchmark.java

Lines changed: 36 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,14 @@
3333
/** Benchmark BLS12-381 G1 and G2 MSM */
3434
public class BLS12Benchmark extends BenchmarkExecutor {
3535

36-
/** Benchmark BLS12-381 G1 and G2 MSM with default warmup and iterations */
37-
public BLS12Benchmark() {
38-
super(MATH_WARMUP, MATH_ITERATIONS);
36+
/**
37+
* The constructor. Use default math based warmup and interations.
38+
*
39+
* @param output where to write the stats.
40+
* @param benchmarkConfig benchmark configurations.
41+
*/
42+
public BLS12Benchmark(final PrintStream output, final BenchmarkConfig benchmarkConfig) {
43+
super(MATH_WARMUP, MATH_ITERATIONS, output, benchmarkConfig);
3944
}
4045

4146
static final String[] scalars = {
@@ -144,8 +149,7 @@ public BLS12Benchmark() {
144149
};
145150

146151
@Override
147-
public void runBenchmark(
148-
final PrintStream output, final Boolean attemptNative, final String fork) {
152+
public void runBenchmark(final Boolean attemptNative, final String fork) {
149153

150154
EvmSpecVersion forkVersion = EvmSpecVersion.fromName(fork);
151155

@@ -179,12 +183,12 @@ private void benchmarkG1Add(final PrintStream output) {
179183
}
180184

181185
BLS12G1AddPrecompiledContract g1addContract = new BLS12G1AddPrecompiledContract();
182-
warmup = MATH_WARMUP / testCases.size();
183-
iterations = MATH_ITERATIONS / testCases.size();
186+
warmIterations = MATH_WARMUP / testCases.size();
187+
execIterations = MATH_ITERATIONS / testCases.size();
184188
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
185189
long gasCost = 0;
186190
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
187-
execTime += runPrecompileBenchmark(testCase.getValue(), g1addContract);
191+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g1addContract);
188192
gasCost += g1addContract.gasRequirement(testCase.getValue());
189193
}
190194
execTime /= testCases.size();
@@ -203,16 +207,16 @@ private void benchmarkG1MultiExp32Pairs(final PrintStream output) {
203207
for (int j = 0; j < 1 << i; j++) {
204208
g1msmPairs.append(g1PointPairs[j]).append(scalars[j]);
205209
}
206-
testCases.put("G1 MSM, " + (1 << i) + " pairs", Bytes.fromHexString(g1msmPairs.toString()));
210+
testCases.put("G1 MSM " + (1 << i) + " pairs", Bytes.fromHexString(g1msmPairs.toString()));
207211
}
208212

209213
BLS12G1MultiExpPrecompiledContract g1msmContract = new BLS12G1MultiExpPrecompiledContract();
210-
warmup = MATH_WARMUP / testCases.size();
211-
iterations = MATH_ITERATIONS / testCases.size();
214+
warmIterations = MATH_WARMUP / testCases.size();
215+
execIterations = MATH_ITERATIONS / testCases.size();
212216
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
213217
long gasCost = 0;
214218
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
215-
execTime += runPrecompileBenchmark(testCase.getValue(), g1msmContract);
219+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g1msmContract);
216220
gasCost += g1msmContract.gasRequirement(testCase.getValue());
217221
}
218222
output.printf(
@@ -227,12 +231,13 @@ private void benchmarkMapFpToG1(final PrintStream output) {
227231
}
228232

229233
BLS12MapFpToG1PrecompiledContract g1MapFpToG1Contract = new BLS12MapFpToG1PrecompiledContract();
230-
warmup = MATH_WARMUP / testCases.size();
231-
iterations = MATH_ITERATIONS / testCases.size();
234+
warmIterations = MATH_WARMUP / testCases.size();
235+
execIterations = MATH_ITERATIONS / testCases.size();
232236
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
233237
long gasCost = 0;
234238
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
235-
execTime += runPrecompileBenchmark(testCase.getValue(), g1MapFpToG1Contract);
239+
execTime +=
240+
runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g1MapFpToG1Contract);
236241
gasCost += g1MapFpToG1Contract.gasRequirement(testCase.getValue());
237242
}
238243
execTime /= testCases.size();
@@ -249,12 +254,12 @@ private void benchmarkG2Add(final PrintStream output) {
249254
}
250255

251256
BLS12G2AddPrecompiledContract g1addContract = new BLS12G2AddPrecompiledContract();
252-
warmup = MATH_WARMUP / testCases.size();
253-
iterations = MATH_ITERATIONS / testCases.size();
257+
warmIterations = MATH_WARMUP / testCases.size();
258+
execIterations = MATH_ITERATIONS / testCases.size();
254259
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
255260
long gasCost = 0;
256261
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
257-
execTime += runPrecompileBenchmark(testCase.getValue(), g1addContract);
262+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g1addContract);
258263
gasCost += g1addContract.gasRequirement(testCase.getValue());
259264
}
260265
execTime /= testCases.size();
@@ -273,16 +278,16 @@ private void benchmarkG2MultiExp32Pairs(final PrintStream output) {
273278
for (int j = 0; j < 1 << i; j++) {
274279
g2msmPairs.append(g2PointPairs[j]).append(scalars[j]);
275280
}
276-
testCases.put("G2 MSM, " + (1 << i) + " pairs", Bytes.fromHexString(g2msmPairs.toString()));
281+
testCases.put("G2 MSM " + (1 << i) + " pairs", Bytes.fromHexString(g2msmPairs.toString()));
277282
}
278283

279284
BLS12G2MultiExpPrecompiledContract g2msmContract = new BLS12G2MultiExpPrecompiledContract();
280-
warmup = MATH_WARMUP / testCases.size();
281-
iterations = MATH_ITERATIONS / testCases.size();
285+
warmIterations = MATH_WARMUP / testCases.size();
286+
execIterations = MATH_ITERATIONS / testCases.size();
282287
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
283288
long gasCost = 0;
284289
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
285-
execTime += runPrecompileBenchmark(testCase.getValue(), g2msmContract);
290+
execTime += runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g2msmContract);
286291
gasCost += g2msmContract.gasRequirement(testCase.getValue());
287292
}
288293
output.printf(
@@ -298,12 +303,13 @@ private void benchmarkMapFp2ToG2(final PrintStream output) {
298303

299304
BLS12MapFp2ToG2PrecompiledContract g1MapFp2ToG2Contract =
300305
new BLS12MapFp2ToG2PrecompiledContract();
301-
warmup = MATH_WARMUP / testCases.size();
302-
iterations = MATH_ITERATIONS / testCases.size();
306+
warmIterations = MATH_WARMUP / testCases.size();
307+
execIterations = MATH_ITERATIONS / testCases.size();
303308
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
304309
long gasCost = 0;
305310
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
306-
execTime += runPrecompileBenchmark(testCase.getValue(), g1MapFp2ToG2Contract);
311+
execTime +=
312+
runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), g1MapFp2ToG2Contract);
307313
gasCost += g1MapFp2ToG2Contract.gasRequirement(testCase.getValue());
308314
}
309315
execTime /= testCases.size();
@@ -322,16 +328,17 @@ private void benchmarkBlsPairing(final PrintStream output) {
322328
for (int j = 0; j < 1 << i; j++) {
323329
pairs.append(g1PointPairs[j]).append(g2PointPairs[j]);
324330
}
325-
testCases.put("BLS Pairing, " + (1 << i) + " pairs", Bytes.fromHexString(pairs.toString()));
331+
testCases.put("BLS Pairing " + (1 << i) + " pairs", Bytes.fromHexString(pairs.toString()));
326332
}
327333

328334
BLS12PairingPrecompiledContract blsPairingContract = new BLS12PairingPrecompiledContract();
329-
warmup = MATH_WARMUP / testCases.size();
330-
iterations = MATH_ITERATIONS / testCases.size();
335+
warmIterations = MATH_WARMUP / testCases.size();
336+
execIterations = MATH_ITERATIONS / testCases.size();
331337
double execTime = Double.MIN_VALUE; // a way to dodge divide by zero
332338
long gasCost = 0;
333339
for (final Map.Entry<String, Bytes> testCase : testCases.entrySet()) {
334-
execTime += runPrecompileBenchmark(testCase.getValue(), blsPairingContract);
340+
execTime +=
341+
runPrecompileBenchmark(testCase.getKey(), testCase.getValue(), blsPairingContract);
335342
gasCost += blsPairingContract.gasRequirement(testCase.getValue());
336343
}
337344

0 commit comments

Comments
 (0)