Merge pull request #104 from baszalmstra/feat/benchmarks

feat: performance benchmarks

.gitignore

@@ -1,4 +1,3 @@
-/target
 /.vscode
 **/*.rs.bk
 Cargo.lock
@@ -29,7 +28,5 @@ Cargo.lock
 .idea/**/libraries
 
 # Build artifacts
-*.dll
-*.dll.lib
-*.so
-*.dylib
+*.munlib
+**/target/*

Cargo.toml

@@ -1,4 +1,4 @@
 [workspace]
 members = [
     "crates/*",
-]
+]

crates/mun_hir/Cargo.toml

@@ -12,7 +12,7 @@ description = "Provides high-level intermediate representation of Mun code"
 salsa="0.12"
 superslice = "1.0"
 mun_syntax={path="../mun_syntax"}
-rustc-hash = "1.0"
+rustc-hash = "1.1"
 once_cell = "0.2"
 relative-path = "0.4.0"
 ena = "0.13"

crates/mun_runtime/Cargo.toml

@@ -18,6 +18,15 @@ notify = "4.0.12"
 parking_lot = "0.10"
 tempfile = "3"
 once_cell = "1.3.1"
+rustc-hash = "1.1"
 
 [dev-dependencies]
 mun_compiler = { path="../mun_compiler" }
+criterion = "0.3"
+termcolor = "1.1"
+mlua = { package ="mlua", version="0.2", default-features = false, features=["vendored", "luajit"]  }
+wasmer-runtime = "0.16"
+
+[[bench]]
+name = "benchmarks"
+harness = false

crates/mun_runtime/benches/benchmarks.rs

@@ -0,0 +1,102 @@
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use mun_runtime::invoke_fn;
+
+mod util;
+
+/// A benchmark test that runs fibonacci(n) for a number of samples and compares the performance
+/// for calling the implementation between different languages.
+pub fn fibonacci_benchmark(c: &mut Criterion) {
+    // Perform setup (not part of the benchmark)
+    let runtime = util::runtime_from_file("fibonacci.mun");
+    let lua = util::lua_from_file("fibonacci.lua");
+    let wasm = util::wasmer_from_file("fibonacci.wasm");
+
+    let mut group = c.benchmark_group("fibonacci");
+
+    // Iterate over a number of samples
+    for i in [100i64, 200i64, 500i64, 1000i64, 4000i64, 8000i64].iter() {
+        // Run Mun fibonacci
+        group.bench_with_input(BenchmarkId::new("mun", i), i, |b, i| {
+            b.iter(|| {
+                let _: i64 = invoke_fn!(runtime, "main", *i).unwrap();
+            })
+        });
+
+        // Run Rust fibonacci
+        group.bench_with_input(BenchmarkId::new("rust", i), i, |b, i| {
+            b.iter(|| fibonacci_main(*i))
+        });
+
+        // Run LuaJIT fibonacci
+        group.bench_with_input(BenchmarkId::new("luajit", i), i, |b, i| {
+            b.iter(|| {
+                let f: mlua::Function = lua.globals().get("main").unwrap();
+                let _: i64 = f.call(*i).unwrap();
+            })
+        });
+
+        // Run Wasm fibonacci
+        group.bench_with_input(BenchmarkId::new("wasm", i), i, |b, i| {
+            b.iter(|| {
+                wasm.call("main", &[(*i as i32).into()]).unwrap();
+            })
+        });
+    }
+
+    group.finish();
+
+    fn fibonacci(n: i64) -> i64 {
+        let mut a = 0;
+        let mut b = 1;
+        let mut i = 1;
+        loop {
+            if i > n {
+                return a;
+            }
+            let sum = a + b;
+            a = b;
+            b = sum;
+            i += 1;
+        }
+    }
+
+    fn fibonacci_main(n: i64) -> i64 {
+        fibonacci(n)
+    }
+}
+
+/// A benchmark method to measure the relative overhead of calling a function from Rust for several
+/// languages.
+pub fn empty_benchmark(c: &mut Criterion) {
+    // Perform setup (not part of the benchmark)
+    let runtime = util::runtime_from_file("empty.mun");
+    let lua = util::lua_from_file("empty.lua");
+    let wasm = util::wasmer_from_file("empty.wasm");
+
+    let mut group = c.benchmark_group("empty");
+
+    group.bench_function("mun", |b| {
+        b.iter(|| {
+            let _: i64 = invoke_fn!(runtime, "empty", black_box(20i64)).unwrap();
+        })
+    });
+    group.bench_function("rust", |b| b.iter(|| empty(black_box(20))));
+    group.bench_function("luajit", |b| {
+        b.iter(|| {
+            let f: mlua::Function = lua.globals().get("empty").unwrap();
+            let _: i64 = f.call(black_box(20)).unwrap();
+        })
+    });
+    group.bench_function("wasm", |b| {
+        b.iter(|| wasm.call("empty", &[black_box(20i64).into()]))
+    });
+
+    group.finish();
+
+    pub fn empty(n: i64) -> i64 {
+        n
+    }
+}
+
+criterion_group!(benches, fibonacci_benchmark, empty_benchmark);
+criterion_main!(benches);

crates/mun_runtime/benches/resources/empty.lua

@@ -0,0 +1,3 @@
+function empty(n)
+    return n
+end

crates/mun_runtime/benches/resources/empty.mun

@@ -0,0 +1,3 @@
+pub fn empty(n:int):int {
+    n
+}

crates/mun_runtime/benches/resources/fibonacci.lua

@@ -0,0 +1,15 @@
+function fibonacci(n)
+    a, b, i = 0,0,0
+    repeat
+        if i > n then
+            return a
+        end
+        sum = a+b
+        a,b = b,sum
+        i = i + 1
+    until false
+end
+
+function main(n)
+    return fibonacci(n)
+end

crates/mun_runtime/benches/resources/fibonacci.mun

@@ -0,0 +1,18 @@
+pub fn fibonacci(n:int):int {
+    let a = 0;
+    let b = 1;
+    let i = 1;
+    loop {
+        if i > n {
+            return a
+        }
+        let sum = a + b;
+        a = b;
+        b = sum;
+        i += 1;
+    }
+}
+
+pub fn main(n:int):int {
+    fibonacci(n)
+}

crates/mun_runtime/benches/util/mod.rs

@@ -0,0 +1,50 @@
+use mlua::Lua;
+use mun_compiler::{Config, Driver, OptimizationLevel, PathOrInline};
+use mun_runtime::RuntimeBuilder;
+use std::cell::RefCell;
+use std::io::Cursor;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+use termcolor::NoColor;
+use wasmer_runtime::{instantiate, Instance};
+
+fn compute_resource_path<P: AsRef<Path>>(p: P) -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("benches/resources/")
+        .join(p)
+}
+
+pub fn runtime_from_file<P: AsRef<Path>>(p: P) -> Rc<RefCell<mun_runtime::Runtime>> {
+    let path = PathOrInline::Path(compute_resource_path(p));
+    let (mut driver, file_id) = Driver::with_file(
+        Config {
+            optimization_lvl: OptimizationLevel::Aggressive,
+            ..Config::default()
+        },
+        path,
+    )
+    .unwrap();
+    let mut cursor = NoColor::new(Cursor::new(Vec::new()));
+    if driver.emit_diagnostics(&mut cursor).unwrap() {
+        let errors = String::from_utf8(cursor.into_inner().into_inner())
+            .unwrap_or_else(|e| format!("<could not utf8 decode error string: {}>", e));
+        panic!("compiler errors..\n{}", errors);
+    }
+
+    let out_path = driver.write_assembly(file_id).unwrap();
+    Rc::new(RefCell::new(RuntimeBuilder::new(out_path).spawn().unwrap()))
+}
+
+pub fn lua_from_file<P: AsRef<Path>>(p: P) -> Lua {
+    let lua = Lua::new();
+    lua.load(&std::fs::read_to_string(compute_resource_path(p)).unwrap())
+        .exec()
+        .unwrap();
+    lua
+}
+
+pub fn wasmer_from_file<P: AsRef<Path>>(p: P) -> Instance {
+    let wasm_content = std::fs::read(compute_resource_path(p)).unwrap();
+    let import_objects = wasmer_runtime::imports! {};
+    instantiate(&wasm_content, &import_objects).unwrap()
+}

crates/mun_runtime/benches/wasm-sources/.gitignore

@@ -0,0 +1 @@
+pkg/

crates/mun_runtime/benches/wasm-sources/Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members=["fibonacci", "empty"]

crates/mun_runtime/benches/wasm-sources/empty/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "empty-wasm"
+version = "0.1.0"
+authors = ["The Mun Team <[email protected]>"]
+edition = "2018"
+homepage = "https://mun-lang.org"
+repository = "https://github.com/mun-lang/mun"
+license = "MIT OR Apache-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wasm-bindgen = "0.2"

crates/mun_runtime/benches/wasm-sources/empty/LICENSE-APACHE

@@ -0,0 +1 @@
+../../../LICENSE-APACHE

crates/mun_runtime/benches/wasm-sources/empty/LICENSE-MIT

@@ -0,0 +1 @@
+../../../LICENSE-MIT

crates/mun_runtime/benches/wasm-sources/empty/src/lib.rs

@@ -0,0 +1,8 @@
+extern crate wasm_bindgen;
+
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+pub fn empty(n: isize) -> isize {
+    n
+}

crates/mun_runtime/benches/wasm-sources/fibonacci/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "fibonacci-wasm"
+version = "0.1.0"
+authors = ["The Mun Team <[email protected]>"]
+edition = "2018"
+homepage = "https://mun-lang.org"
+repository = "https://github.com/mun-lang/mun"
+license = "MIT OR Apache-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wasm-bindgen = "0.2"

crates/mun_runtime/benches/wasm-sources/fibonacci/LICENSE-APACHE

@@ -0,0 +1 @@
+../../../LICENSE-APACHE

crates/mun_runtime/benches/wasm-sources/fibonacci/LICENSE-MIT

@@ -0,0 +1 @@
+../../../LICENSE-MIT

crates/mun_runtime/benches/wasm-sources/fibonacci/src/lib.rs

@@ -0,0 +1,23 @@
+extern crate wasm_bindgen;
+
+use wasm_bindgen::prelude::*;
+
+fn fibonacci(n: isize) -> isize {
+    let mut a = 0;
+    let mut b = 1;
+    let mut i = 1;
+    loop {
+        if i > n {
+            return a;
+        }
+        let sum = a + b;
+        a = b;
+        b = sum;
+        i += 1;
+    }
+}
+
+#[wasm_bindgen]
+pub fn main(n: isize) -> isize {
+    fibonacci(n)
+}

crates/mun_runtime/src/lib.rs

@@ -8,15 +8,14 @@ mod assembly;
 mod function;
 #[macro_use]
 mod macros;
+#[macro_use]
+mod type_info;
 mod garbage_collector;
 mod marshal;
 mod reflection;
 mod static_type_map;
 mod r#struct;
 
-#[macro_use]
-mod type_info;
-
 #[cfg(test)]
 mod test;
 
@@ -40,6 +39,7 @@ use crate::function::IntoFunctionInfo;
 pub use crate::r#struct::StructRef;
 use garbage_collector::GarbageCollector;
 use gc::GcRuntime;
+use rustc_hash::FxHashMap;
 use std::sync::Arc;
 
 impl_has_type_info_name!(
@@ -103,7 +103,7 @@ impl RuntimeBuilder {
 /// A runtime dispatch table that maps full paths to function and struct information.
 #[derive(Default)]
 pub struct DispatchTable {
-    functions: HashMap<String, abi::FunctionInfo>,
+    functions: FxHashMap<String, abi::FunctionInfo>,
 }
 
 impl DispatchTable {

crates/mun_runtime/src/macros.rs

@@ -98,7 +98,7 @@ macro_rules! invoke_fn_impl {
                     match runtime
                         .borrow()
                         .get_function_info(function_name)
-                        .ok_or(format!("Failed to obtain function '{}'", function_name))
+                        .ok_or_else(|| format!("Failed to obtain function '{}'", function_name))
                         .and_then(|function_info| {
                             // Validate function signature
                             let num_args = $crate::count_args!($($T),*);