Benchmark interpret vs compiled code evaluation

.gitignore

@@ -21,4 +21,4 @@ docs/site/
 # It records a fixed state of all packages used by the project. As such, it should not be
 # committed for packages, but should be committed for applications that require a static
 # environment.
-Manifest.toml
+**/Manifest.toml

benchmark/.gitignore

@@ -0,0 +1 @@
+output/

benchmark/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7"
+Metatheory = "e9d8d322-4543-424a-9be4-0cc815abe26c"

benchmark/benchmarks.jl

@@ -0,0 +1,86 @@
+using Pkg
+Pkg.activate(@__DIR__) # activate the benchmark environment
+Pkg.instantiate()
+
+using BenchmarkTools
+using HerbInterpret
+@warn "About to load Metatheory. Currently, the 3.0 development branch claims to be much faster. Consider installing it."
+using Metatheory
+
+const SUITE = BenchmarkGroup()
+
+SUITE["interpret"] = BenchmarkGroup()
+
+function example_function(input1)
+    if input1 % 5 == 0 && input1 % 3 == 0
+        return "FizzBuzz"
+    elseif input1 % 3 == 0
+        return "Fizz"
+    elseif input1 % 5 == 0
+        return "Buzz"
+    else
+        return string(input1)
+    end
+end
+
+tab = Dict{Symbol, Any}(
+    :%      => rem,
+    :(==)   => ==,
+    :string => string,
+    :Int    => Int64,
+    :String => String,
+    :input1 => 15,
+    :example_function => example_function
+)
+
+Mem = Dict{Symbol,Union{Bool,Int}}
+read_mem = @theory v σ begin
+  (v::Symbol, σ::Mem) => σ[v]
+end
+
+σ₁ = Mem(:x => 2)
+program = :(x, $σ₁)
+
+benchmark_outputs = joinpath(@__DIR__, "output")
+
+if !isdir(benchmark_outputs) mkdir(benchmark_outputs) end
+
+@info "Benchmarking Compiled Version"
+SUITE["interpret"] = @benchmarkable example_function(15)
+tune!(SUITE)
+results = run(SUITE; verbose=true)
+BenchmarkTools.save(joinpath(benchmark_outputs, "bench-compiled.json"), results)
+
+println("Sleeping for 5s to relax...")
+sleep(5)
+
+@info "Benchmarking with interpreter from `HerbInterpret`"
+SUITE["interpret"] = @benchmarkable interpret(tab, :(example_function(input1)))
+tune!(SUITE)
+results = run(SUITE; verbose=true)
+BenchmarkTools.save(joinpath(benchmark_outputs, "bench-interpret.json"), results)
+
+println("Sleeping for 5s to relax...")
+sleep(5)
+
+@info "Benchmarking With Metatheory"
+SUITE["interpret"] = @benchmarkable rewrite(program, read_mem)
+tune!(SUITE)
+results = run(SUITE; verbose=true)
+BenchmarkTools.save(joinpath(benchmark_outputs, "bench-metatheory.json"), results)
+
+results = Dict([basename(name)[7:end-5] => BenchmarkTools.load(name) for name in readdir(benchmark_outputs, join=true)])
+
+interpret_vs_compiled = judge(
+    mean(results["interpret"][1]),
+    mean(results["compiled"][1]),
+)["interpret"]
+
+@show interpret_vs_compiled
+
+metatheory_vs_compiled = judge(
+    mean(results["metatheory"][1]),
+    mean(results["compiled"][1]),
+)["interpret"]
+
+@show metatheory_vs_compiled