memory64 support (#37)

* Adds support for the memory64 spec proposal. table64 support is TODO. * New overridable logging interface. Exposure via FFI TODO. * Reorganized tests a bit to ensure the wasm testsuite doesn't get mixed up with the other tests * New mem64 test that builds a C program into wasm64 with clang and ensures it can access 64-bit addresses * Readme updates to better document test requirements/dependencies * Fixed a bug in constexpr funcref resolution where the module instance wasn't correctly set on resolution
rdunnington · May 19, 2024 · 1e4fe3f · 1e4fe3f
1 parent ce02943
commit 1e4fe3f
Show file tree

Hide file tree

Showing 21 changed files with 765 additions and 248 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,6 +1,10 @@
 name: CI
 
-on: [push, pull_request]
+on: 
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   tests:
@@ -18,19 +22,31 @@ jobs:
         with:
           version: 0.11.0
 
+      # The current default version of clang on macos runners is 14, which doesn't support the wasm64-freestanding target.
+      - name: Install LLVM and Clang
+        if: matrix.os == 'macos-latest'
+        uses: KyleMayes/install-llvm-action@v2
+        with:
+          version: "15.0"
+
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
           cache: pip
 
-      - name: Install dependencies
+      - name: Install python dependencies
         working-directory: test/wasi/wasi-testsuite/test-runner
         run: python3 -m pip install -r requirements.txt
 
-      - uses: mwilliamson/setup-wabt-action@v2
-        with:
-          wabt-version: "1.0.31"
+      # Ideally we would use this but it seems to be broken
+      # - name: Setup wasm-tools 
+      #   uses: jcbhmr/setup-wasm-tools@v2
+      #   with:
+      #     wasm-tools-version: 1.207
+
+      - name: Setup wasm-tools
+        run: cargo install wasm-tools
 
       - name: Build
         run: |
@@ -44,6 +60,10 @@ jobs:
         run: |
           zig build test-wasm -- --log-suite
 
+      - name: Run mem64 test
+        run: |
+          zig build test-mem64
+
       - name: Run wasi testsuite
         run: |
           zig build test-wasi
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,11 @@
 zig-cache
 zig-out
-test/wasm
+test/wasm/wasm-generated
 *.wasm
 *.wasm.o
 .vs
 *.sublime-project
 *.sublime-workspace
 *.sln
-*.rdbg
+*.rdbg
+.DS_Store
diff --git a/.gitmodules b/.gitmodules
@@ -2,7 +2,7 @@
 	path = zig-stable-array
 	url = https://github.com/rdunnington/zig-stable-array.git
 [submodule "test/testsuite"]
-	path = test/testsuite
+	path = test/wasm/wasm-testsuite
 	url = https://github.com/WebAssembly/testsuite.git
 [submodule "test/wasi/wasi-testsuite"]
 	path = test/wasi/wasi-testsuite

diff --git a/README.md b/README.md
@@ -3,25 +3,32 @@
 <div align="center">
 <a href=https://webassembly.org/><img src="https://avatars.githubusercontent.com/u/11578470?s=200&v=4" alt="Markdown Logo" width="150"/></a>
 
-Bytebox is a Webassembly VM.
+Bytebox is a WebAssembly VM.
 </div>
 
-## Getting started
+# Getting started
 
-### Requirements
+## Requirements
 Bytebox currently builds with [Zig 0.11.x](https://ziglang.org/download) to avoid churn on zig master.
 
-### Run
+To run the tests:
+* `wasm-tools` is required to run the wasm testsuite. You can install it via the rust toolchain `cargo install wasm-tools` or directly from the [release page](https://github.com/bytecodealliance/wasm-tools/releases).
+* `clang` v15.x+ is required to build the mem64 tests. However, if you don't have a compatible version of `clang` installed, you can pass `--noclang` to `zig build` to avoid the requirement.
+* `python3` is required to run the wasi testsuite. You may need to run `python3 -m pip install -r test/wasi/wasi-testsuite/test-runner/requirements.txt` to ensure the wasi test runner has all the necessary dependencies installed.
+
+## Run Tests
 
 ```sh
 git clone --recurse-submodules https://github.com/rdunnington/bytebox.git
 cd bytebox
-zig build test  # run the WebAssembly spec testsuite
-# run the wasi testsuite
-python3 test/wasi/wasi-testsuite/test-runner/wasi_test_runner.py -r test/wasi/bytebox_adapter.py -t ./test/wasi/wasi-testsuite/tests/assemblyscript/testsuite/ ./test/wasi/wasi-testsuite/tests/c/testsuite/ ./test/wasi/wasi-testsuite/tests/rust/testsuite/
+zig build test-unit # run builtin zig unit tests
+zig build test-wasm # run official wasm spec testsuite
+zig build test-wasi # run official wasi spec testsuite
+zig build test-mem64 # run memory64 compat test
+zig build test # run all of the above in parallel (output will not be pretty!)
 ```
 
-### Usage
+## Usage
 
 You can use the standalone runtime to load and execute WebAssembly programs:
 ```sh
@@ -56,22 +63,22 @@ pub fn main() !void {
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     var allocator: std.mem.Allocator = gpa.allocator();
 
-    var wasm_data: []u8 = try std.fs.cwd().readFileAlloc(allocator, "example.wasm", 1024 * 128);
+    const wasm_data: []u8 = try std.fs.cwd().readFileAlloc(allocator, "example.wasm", 1024 * 128);
     defer allocator.free(wasm_data);
 
-    var module_definition = bytebox.ModuleDefinition.init(allocator, .{});
-    defer module_definition.deinit();
-    try module_definition.decode(wasm_data);
+    const module_def = try bytebox.createModuleDefinition(allocator, .{});
+    defer module_def.destroy();
+    try module_def.decode(wasm_data);
 
-    var module_instance = bytebox.ModuleInstance.init(&module_definition, allocator);
-    defer module_instance.deinit();
+    const module_instance = try bytebox.createModuleInstance(.Stack, module_def, allocator);
+    defer module_instance.destroy();
     try module_instance.instantiate(.{});
 }
 ```
 
 Inter-language FFI is also supported. See `src/bytebox.h` for an overview in C. To use bytebox as a static library, link with the built library in `zig-out/lib/`. Note that Zig assumes a default stack size of 8MB, so you'll need to ensure the same in your program.
 
-## Status
+# Status
 
 This project is still in the alpha stage.
 
@@ -81,7 +88,7 @@ This project is still in the alpha stage.
 |❌|TODO|
 |💀|Not planned/Removed from spec|
 
-### [WebAssembly](https://webassembly.github.io/spec/core/index.html) support:
+## [WebAssembly](https://webassembly.github.io/spec/core/index.html) support:
 
 | Status | Feature |
 | --- | --- |
@@ -95,7 +102,7 @@ This project is still in the alpha stage.
 |✅|Bulk memory and table instructions|
 |✅|Vector instructions|
 
-### [WASI Preview 1](https://github.com/WebAssembly/WASI/tree/main) support:
+## [WASI Preview 1](https://github.com/WebAssembly/WASI/tree/main) support:
 
 | Status | Feature |
 | --- | --- |

diff --git a/build.zig b/build.zig
@@ -17,14 +17,15 @@ const ExeOpts = struct {
 
 pub fn build(b: *Build) void {
     const should_emit_asm = b.option(bool, "asm", "Emit asm for the bytebox binaries") orelse false;
+    const no_clang = b.option(bool, "noclang", "Pass this if clang isn't in the PATH") orelse false;
+
+    var bench_add_one_step: *CompileStep = buildWasmLib(b, "bench/samples/add-one.zig");
+    var bench_fibonacci_step: *CompileStep = buildWasmLib(b, "bench/samples/fibonacci.zig");
+    var bench_mandelbrot_step: *CompileStep = buildWasmLib(b, "bench/samples/mandelbrot.zig");
 
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
-    var bench_add_one_step: *CompileStep = buildWasmLib(b, "bench/samples/add-one.zig", optimize);
-    var bench_fibonacci_step: *CompileStep = buildWasmLib(b, "bench/samples/fibonacci.zig", optimize);
-    var bench_mandelbrot_step: *CompileStep = buildWasmLib(b, "bench/samples/mandelbrot.zig", optimize);
-
     const bytebox_module: *Build.Module = b.addModule("bytebox", .{
         .source_file = Build.LazyPath.relative("src/core.zig"),
     });
@@ -43,7 +44,7 @@ pub fn build(b: *Build) void {
         &bench_mandelbrot_step.step,
     };
     _ = buildExeWithRunStep(b, target, optimize, bytebox_module, .{
-        .exe_name = "benchmark",
+        .exe_name = "bench",
         .root_src = "bench/main.zig",
         .step_name = "bench",
         .description = "Run the benchmark suite",
@@ -71,8 +72,8 @@ pub fn build(b: *Build) void {
 
     // wasm tests
     var wasm_testsuite_step = buildExeWithRunStep(b, target, optimize, bytebox_module, .{
-        .exe_name = "testsuite",
-        .root_src = "test/main.zig",
+        .exe_name = "test-wasm",
+        .root_src = "test/wasm/main.zig",
         .step_name = "test-wasm",
         .description = "Run the wasm testsuite",
     });
@@ -83,11 +84,43 @@ pub fn build(b: *Build) void {
     const wasi_testsuite_step = b.step("test-wasi", "Run wasi testsuite");
     wasi_testsuite_step.dependOn(&wasi_testsuite.step);
 
+    // mem64 step
+    var mem64_test_step: ?*Build.Step = null;
+    if (!no_clang) {
+        // need to use clang to compile the C test due to https://github.com/ziglang/zig/issues/19942
+        // eventually we will ziggify this test
+        // ideally this test would go away, but the existing spec tests don't provide very good coverage
+        // of the instructions
+        const compile_memtest = b.addSystemCommand(&.{"clang"});
+        compile_memtest.addArg("--target=wasm64-freestanding");
+        compile_memtest.addArg("-mbulk-memory");
+        compile_memtest.addArg("-nostdlib");
+        compile_memtest.addArg("-O2");
+        compile_memtest.addArg("-Wl,--no-entry");
+        compile_memtest.addArg("-Wl,--export-dynamic");
+        compile_memtest.addArg("-o");
+        compile_memtest.addArg("test/mem64/memtest.wasm");
+        compile_memtest.addFileArg(.{ .path = "test/mem64/memtest.c" });
+        compile_memtest.has_side_effects = true;
+
+        b.getInstallStep().dependOn(&compile_memtest.step);
+
+        mem64_test_step = buildExeWithRunStep(b, target, optimize, bytebox_module, .{
+            .exe_name = "test-mem64",
+            .root_src = "test/mem64/main.zig",
+            .step_name = "test-mem64",
+            .description = "Run the mem64 test",
+        });
+    }
+
     // All tests
     const all_tests_step = b.step("test", "Run unit, wasm, and wasi tests");
     all_tests_step.dependOn(unit_test_step);
     all_tests_step.dependOn(wasm_testsuite_step);
     all_tests_step.dependOn(wasi_testsuite_step);
+    if (mem64_test_step) |step| {
+        all_tests_step.dependOn(step);
+    }
 }
 
 fn buildExeWithRunStep(b: *Build, target: CrossTarget, optimize: std.builtin.Mode, bytebox_module: *Build.Module, opts: ExeOpts) *Build.Step {
@@ -100,10 +133,6 @@ fn buildExeWithRunStep(b: *Build, target: CrossTarget, optimize: std.builtin.Mod
 
     exe.addModule("bytebox", bytebox_module);
 
-    // exe.addModule("bytebox", .{
-    //     .source_file = Build.LazyPath.relative("src/core.zig"),
-    // });
-
     // exe.emit_asm = if (opts.should_emit_asm) .emit else .default;
     b.installArtifact(exe);
 
@@ -125,7 +154,7 @@ fn buildExeWithRunStep(b: *Build, target: CrossTarget, optimize: std.builtin.Mod
     return step;
 }
 
-fn buildWasmLib(b: *Build, filepath: []const u8, optimize: std.builtin.Mode) *CompileStep {
+fn buildWasmLib(b: *Build, filepath: []const u8) *CompileStep {
     var filename: []const u8 = std.fs.path.basename(filepath);
     var filename_no_extension: []const u8 = filename[0 .. filename.len - 4];
 
@@ -136,11 +165,9 @@ fn buildWasmLib(b: *Build, filepath: []const u8, optimize: std.builtin.Mode) *Co
             .cpu_arch = .wasm32,
             .os_tag = .freestanding,
         },
-        .optimize = optimize,
+        .optimize = .ReleaseSmall,
     });
 
-    // const mode = b.standardOptimizeOption();
-    // lib.setBuildMode(mode);
     b.installArtifact(lib);
 
     return lib;