benchmark: measure timings in microseconds

benchmark/compare.py

@@ -13,7 +13,7 @@ def load_data(btype: str, dname: str, providers: list[str], results_dir: str) ->
         with open(f'{results_dir}/{pname}.{btype}_{dname}.json', 'r') as fh:
             data.append(json.load(fh))
         total_time = sum(ts if ts > 0 else 1 for (ts, _) in data[-1].values())
-        times.append(total_time / 1000.)
+        times.append(total_time / 1_000_000.) # us -> sec
     return data, times
 
 def process_selectors(dname: str, providers: list[str], results_dir: str):

benchmark/providers/etherscan/main.py

@@ -155,8 +155,8 @@ def process(data, mode):
         d = json.load(fh)
         t0 = time.perf_counter()
         r = process(d, mode)
-        duration_ms = int((time.perf_counter() - t0) * 1000)
-        ret[fname] = [duration_ms, r]
+        duration_us = int(time.perf_counter() - t0)
+        ret[fname] = [duration_us, r]
 
 with open(outfile, 'w') as fh:
     json.dump(ret, fh)

benchmark/providers/ethersolve/src/main/java/HelloEtherSolve.java

@@ -51,9 +51,9 @@ private void processFile(Path file, Map<String, Object[]> results) throws Except
 
         long startTime = System.nanoTime();
         List<Long[]> processResults = executeWithTimeout(bytecode);
-        long timeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+        long timeUs = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startTime);
 
-        results.put(file.getFileName().toString(), new Object[]{timeMs, processResults});
+        results.put(file.getFileName().toString(), new Object[]{timeUs, processResults});
     }
 
     public void execute(String[] args) throws Exception {

benchmark/providers/evm-cfg-builder/main.py

@@ -13,15 +13,15 @@ def extract_cfg(code_hex: str):
         cfg = CFG(code_hex)
     except Exception as e:
         print(e)
-        duration_ms = int((time.monotonic() - start_ts) * 1000)
-        return [duration_ms, []]
+        duration_us = int(time.monotonic() - start_ts)
+        return [duration_us, []]
 
-    duration_ms = int((time.monotonic() - start_ts) * 1000)
+    duration_us = int(time.monotonic() - start_ts)
     for x in cfg.basic_blocks:
         assert all(ins.mnemonic != 'JUMPDEST' for ins in x.instructions[1:]), x.instructions
     result = [(basic_block.start.pc, out.start.pc) for basic_block in cfg.basic_blocks for out in basic_block.all_outgoing_basic_blocks]
 
-    return [duration_ms, sorted(result)]
+    return [duration_us, sorted(result)]
 
 
 if len(sys.argv) < 4:

benchmark/providers/evm-cfg/main.py

@@ -56,17 +56,17 @@ def process_fast(output: str) -> list:
 def extract_cfg(code_hex: str):
     start_ts = time.monotonic()
     try:
-        output = subprocess.check_output(['evm-cfg', code_hex], timeout=10, text=True)
+        output = subprocess.check_output(['evm-cfg', code_hex], timeout=15, text=True)
     except Exception as e:
         print('Err')
-        duration_ms = int((time.monotonic() - start_ts) * 1000)
-        return (duration_ms, [])
-    duration_ms = int((time.monotonic() - start_ts) * 1000)
+        duration_us = int(time.monotonic() - start_ts)
+        return (duration_us, [])
+    duration_us = int(time.monotonic() - start_ts)
 
     # ret = process_slow(output)
     ret = process_fast(output)
 
-    return [duration_ms, sorted(ret)]
+    return [duration_us, sorted(ret)]
 
 
 if len(sys.argv) < 4:

benchmark/providers/evm-hound-rs/src/main.rs

@@ -23,7 +23,7 @@ fn main() -> std::io::Result<()> {
         std::process::exit(1);
     }
 
-    type Meta = u64; // duration in ms
+    type Meta = u64; // duration in us
     let mut ret: HashMap<String, (Meta, Vec<String>)> = HashMap::new();
 
     for entry in fs::read_dir(indir)? {
@@ -40,10 +40,10 @@ fn main() -> std::io::Result<()> {
 
         let now = Instant::now();
         let r = evm_hound::selectors_from_bytecode(&code);
-        let duration_ms = now.elapsed().as_millis() as u64;
+        let duration_us = now.elapsed().as_micros() as u64;
         let string_selectors: Vec<_> = r.into_iter().map(hex::encode).collect();
 
-        ret.insert(fname, (duration_ms, string_selectors));
+        ret.insert(fname, (duration_us, string_selectors));
     }
 
     let file = fs::File::create(outfile)?;

benchmark/providers/evmole-js/main.mjs

@@ -18,28 +18,28 @@ const selectors = mode === 'mutability' || mode === 'arguments' ? JSON.parse(rea
 function timeit(fn) {
   const start_ts = hrtime.bigint();
   const r = fn();
-  const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n);
-  return [duration_ms, r]
+  const duration_us = Number((hrtime.bigint() - start_ts) / 1000n);
+  return [duration_us, r]
 }
 
 function extract(code, mode, fname) {
   if (mode === 'selectors') {
-    let [duration_ms, r] = timeit(() => contractInfo(code, {selectors: true}));
-    return [duration_ms, r.functions.map((f) => f.selector)];
+    let [duration_us, r] = timeit(() => contractInfo(code, {selectors: true}));
+    return [duration_us, r.functions.map((f) => f.selector)];
   } else if (mode === 'arguments') {
-    let [duration_ms, r] = timeit(() => contractInfo(code, {arguments: true}));
+    let [duration_us, r] = timeit(() => contractInfo(code, {arguments: true}));
     const by_sel = new Map(r.functions.map((f) => [f.selector, f.arguments]));
-    return [duration_ms, Object.fromEntries(
+    return [duration_us, Object.fromEntries(
       selectors[fname][1].map((s) => [s, by_sel.get(s) ?? 'notfound'])
     )];
   } else if (mode === 'mutability') {
-    let [duration_ms, r] = timeit(() => contractInfo(code, {stateMutability: true}));
+    let [duration_us, r] = timeit(() => contractInfo(code, {stateMutability: true}));
     const by_sel = new Map(r.functions.map((f) => [f.selector, f.stateMutability]));
-    return [duration_ms, Object.fromEntries(
+    return [duration_us, Object.fromEntries(
       selectors[fname][1].map((s) => [s, by_sel.get(s) ?? 'notfound'])
     )];
   } else if (mode === 'flow') {
-    let [duration_ms, r] = timeit(() => contractInfo(code, {controlFlowGraph: true}));
+    let [duration_us, r] = timeit(() => contractInfo(code, {controlFlowGraph: true}));
     let ret = []
     for (const b of r.controlFlowGraph.blocks) {
       let bt = b.get('type');
@@ -69,7 +69,7 @@ function extract(code, mode, fname) {
         throw `unknown block type ${bt}`;
       }
     }
-    return [duration_ms, ret];
+    return [duration_us, ret];
   } else {
     throw 'unsupported mode';
   }

benchmark/providers/evmole-py/main.py

@@ -33,7 +33,7 @@
             info = contract_info(code, control_flow_graph=True)
         else:
             raise Exception(f'Unknown mode {cfg.mode}')
-        duration_ms = int((time.perf_counter() - t0) * 1000)
+        duration_us = int((time.perf_counter() - t0))
 
         if cfg.mode == 'selectors':
             r = [f.selector for f in info.functions]
@@ -66,7 +66,7 @@
         else:
             raise Exception(f'Unknown mode {cfg.mode}')
 
-        ret[fname] = [duration_ms, r]
+        ret[fname] = [duration_us, r]
 
 with open(cfg.output_file, 'w') as fh:
     json.dump(ret, fh)

benchmark/providers/evmole-rs/src/main.rs

@@ -45,8 +45,8 @@ fn timeit(args: evmole::ContractInfoArgs) -> (evmole::Contract, u64)
 {
     let now = Instant::now();
     let result = evmole::contract_info(args);
-    let duration = now.elapsed().as_millis() as u64;
-    (result, duration)
+    let duration_us = now.elapsed().as_micros() as u64;
+    (result, duration_us)
 }
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {

benchmark/providers/heimdall-rs/src/main.rs

@@ -31,11 +31,21 @@ struct Args {
     selectors_file: Option<String>,
 }
 
+async fn measure_time<T, F>(f: F) -> (T, u64)
+where
+    F: std::future::Future<Output = T>,
+{
+    let start = Instant::now();
+    let result = f.await;
+    let duration_us = start.elapsed().as_micros() as u64;
+    (result, duration_us)
+}
+
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cfg = Args::parse();
 
-    type Meta = u64; // duration in ms
+    type Meta = u64; // duration in us
 
     let selectors: HashMap<String, (Meta, Vec<String>)> = match cfg.mode {
         Mode::Selectors | Mode::Flow => HashMap::new(),
@@ -64,9 +74,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                     .target(hex_code)
                     .skip_resolving(true)
                     .build()?;
-                let now = Instant::now();
-                let result = heimdall_core::heimdall_decompiler::decompile(dargs).await;
-                let duration_ms = now.elapsed().as_millis() as u64;
+
+                let (result, duration_us) = measure_time(heimdall_core::heimdall_decompiler::decompile(dargs)).await;
+
                 let r = match result {
                     Err(e) => {
                         println!("got error for {}: {}", fname, e);
@@ -79,16 +89,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         .map(|x| x.strip_prefix("Unresolved_").unwrap().to_string())
                         .collect(),
                 };
-                ret_selectors.insert(fname, (duration_ms, r));
+                ret_selectors.insert(fname, (duration_us, r));
             }
             Mode::Arguments => {
                 let dargs = DecompilerArgsBuilder::new()
                     .target(hex_code)
                     .skip_resolving(true)
                     .build()?;
-                let now = Instant::now();
-                let result = heimdall_core::heimdall_decompiler::decompile(dargs).await;
-                let duration_ms = now.elapsed().as_millis() as u64;
+
+                let (result, duration_us) = measure_time(heimdall_core::heimdall_decompiler::decompile(dargs)).await;
+
                 let r = match result {
                     Err(e) => {
                         println!("got error for {}: {}", fname, e);
@@ -127,16 +137,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                             .collect()
                     }
                 };
-                ret_other.insert(fname, (duration_ms, r));
+                ret_other.insert(fname, (duration_us, r));
             }
             Mode::Mutability => {
                 let dargs = DecompilerArgsBuilder::new()
                     .target(hex_code)
                     .skip_resolving(true)
                     .build()?;
-                let now = Instant::now();
-                let result = heimdall_core::heimdall_decompiler::decompile(dargs).await;
-                let duration_ms = now.elapsed().as_millis() as u64;
+
+                let (result, duration_us) = measure_time(heimdall_core::heimdall_decompiler::decompile(dargs)).await;
+
                 let r = match result {
                     Err(e) => {
                         println!("got error for {}: {}", fname, e);
@@ -174,16 +184,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                             .collect()
                     }
                 };
-                ret_other.insert(fname, (duration_ms, r));
+                ret_other.insert(fname, (duration_us, r));
             }
             Mode::Flow => {
                 let cfg_args = heimdall_core::heimdall_cfg::CfgArgsBuilder::new()
                     .target(hex_code)
                     .build()?;
 
-                let now = Instant::now();
-                let cfg = heimdall_core::heimdall_cfg::cfg(cfg_args).await?;
-                let duration_ms = now.elapsed().as_millis() as u64;
+                let (result, duration_us) = measure_time(heimdall_core::heimdall_cfg::cfg(cfg_args)).await;
+                let cfg = result?;
 
                 let mut jump_dest_mapping: HashMap<usize, usize> = HashMap::new();
                 let mut control_flow: BTreeSet<(usize, usize)> = BTreeSet::new();
@@ -231,7 +240,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                     (from, target)
                 }));
 
-                ret_flow.insert(fname, (duration_ms, control_flow));
+                ret_flow.insert(fname, (duration_us, control_flow));
             }
         }
     }

benchmark/providers/sevm/main.mjs

@@ -18,22 +18,22 @@ const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {}
 function timeit(fn) {
   const start_ts = hrtime.bigint();
   const r = fn();
-  const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n);
-  return [duration_ms, r]
+  const duration_us = Number((hrtime.bigint() - start_ts) / 1000n);
+  return [duration_us, r]
 }
 
 function extract(code, mode, fname) {
-  const [duration_ms, contract] = timeit(() => {
+  const [duration_us, contract] = timeit(() => {
     try {
       return new Contract(code);
     } catch (e) {
       // console.log(e);
     }
   });
   if (mode === 'selectors') {
-    return [duration_ms, Object.keys(contract ? contract.functions : {})]
+    return [duration_us, Object.keys(contract ? contract.functions : {})]
   } else if (mode === 'mutability') {
-    return [duration_ms, Object.fromEntries(selectors[fname][1].map((s) => {
+    return [duration_us, Object.fromEntries(selectors[fname][1].map((s) => {
       const fn = contract ? contract.functions[s] : undefined;
       if (fn === undefined) {
         return [s, 'selnotfound'];
@@ -71,7 +71,7 @@ function extract(code, mode, fname) {
         }
       }
     }
-    return [duration_ms, Array.from(res.values())];
+    return [duration_us, Array.from(res.values())];
   } else {
     throw 'unsupported mode';
   }

benchmark/providers/simple/main.py

@@ -45,8 +45,8 @@ def extract_selectors(code: bytes) -> list[str]:
             r = extract_selectors(code)
         else:
             raise Exception(f'Unknown mode {mode}')
-        duration_ms = int((time.perf_counter() - t0) * 1000)
-        ret[fname] = [duration_ms, r]
+        duration_us = int(time.perf_counter() - t0)
+        ret[fname] = [duration_us, r]
 
 with open(outfile, 'w') as fh:
     json.dump(ret, fh)

benchmark/providers/smlxl/src/main.rs

@@ -98,7 +98,7 @@ impl Watchdog for MyWatchDog {
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cfg = Args::parse();
-    type Meta = u64; // duration in ms
+    type Meta = u64; // duration in us
     let mut ret_other: HashMap<String, (Meta, HashMap<String, String>)> = HashMap::new();
     for entry in fs::read_dir(cfg.input_dir)? {
         let entry = entry?;
@@ -125,7 +125,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         let extractor = storage_layout_extractor::new(contract, vm_config, unifier_config, watchdog);
         let now = Instant::now();
         let r = extractor.analyze();
-        let dur = now.elapsed().as_millis() as u64;
+        let dur = now.elapsed().as_micros() as u64;
 
         ret_other.insert(
             fname,

benchmark/providers/whatsabi/main.mjs

@@ -18,18 +18,18 @@ const selectors = mode === 'mutability' ? JSON.parse(readFileSync(argv[5])) : {}
 function timeit(fn) {
   const start_ts = hrtime.bigint();
   const r = fn();
-  const duration_ms = Number((hrtime.bigint() - start_ts) / 1000000n);
-  return [duration_ms, r]
+  const duration_us = Number((hrtime.bigint() - start_ts) / 1000n);
+  return [duration_us, r]
 }
 
 function extract(code, mode, fname) {
   if (mode === 'selectors') {
-    const [duration_ms, r] = timeit(() => whatsabi.selectorsFromBytecode(code))
-    return [duration_ms, r.map(x => x.slice(2))]; // remove '0x' prefix
+    const [duration_us, r] = timeit(() => whatsabi.selectorsFromBytecode(code))
+    return [duration_us, r.map(x => x.slice(2))]; // remove '0x' prefix
   } else if (mode === 'mutability') {
-    const [duration_ms, abi] = timeit(() => whatsabi.abiFromBytecode(code));
+    const [duration_us, abi] = timeit(() => whatsabi.abiFromBytecode(code));
     const smut = Object.fromEntries(abi.filter((v) => v.type == 'function').map((v) => [v.selector, v.stateMutability]));
-    return [duration_ms, Object.fromEntries(selectors[fname][1].map((s) => [s, smut[`0x${s}`] || 'selnotfound']))];
+    return [duration_us, Object.fromEntries(selectors[fname][1].map((s) => [s, smut[`0x${s}`] || 'selnotfound']))];
   } else {
     throw 'unsupported mode';
   }