feat： Add local support for zkm build (#190)

* feat: add initial support for zkm-build

* add log and ported information

* update revme example

* update tests in example

* add missed Cargo.toml

* fix and update README

* fix clippy

* update dir structure of example tests

* update

* remove zkmips.rs

Cargo.toml

@@ -2,7 +2,8 @@
 members = [
   "runtime/*",
   "emulator",
-  "prover"
+  "prover",
+  "build",
 ]
 resolver = "2"
 

build/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "zkm-build"
+description = "Build an ZKM program."
+readme = "README.md"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+cargo_metadata = "0.18.1"
+anyhow = { version = "1.0.83" }
+clap = { version = "4.5.9", features = ["derive", "env"] }
+dirs = "5.0.1"
+chrono = { version = "0.4.38", default-features = false, features = ["clock"] }

build/README.md

@@ -0,0 +1,12 @@
+# zkm-build
+Lightweight crate used to build ZKM programs.
+
+Exposes `build_program`, which builds an ZKM program in the local environment or in a docker container with the specified parameters from `BuildArgs`.
+
+## Usage
+
+```rust
+use zkm_build::build_program;
+
+build_program(&BuildArgs::default(), Some(program_dir));
+```

build/src/build.rs

@@ -0,0 +1,106 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use cargo_metadata::camino::Utf8PathBuf;
+
+use crate::{
+    command::{local::create_local_command, utils::execute_command},
+    utils::{cargo_rerun_if_changed, copy_elf_to_output_dir, current_datetime},
+    BuildArgs,
+};
+
+/// Build a program with the specified [`BuildArgs`]. The `program_dir` is specified as an argument
+/// when the program is built via `build_program`.
+///
+/// # Arguments
+///
+/// * `args` - A reference to a `BuildArgs` struct that holds various arguments used for building
+///   the program.
+/// * `program_dir` - An optional `PathBuf` specifying the directory of the program to be built.
+///
+/// # Returns
+///
+/// * `Result<Utf8PathBuf>` - The path to the built program as a `Utf8PathBuf` on success, or an
+///   error on failure.
+pub fn execute_build_program(
+    args: &BuildArgs,
+    program_dir: Option<PathBuf>,
+) -> Result<Utf8PathBuf> {
+    // If the program directory is not specified, use the current directory.
+    let program_dir = program_dir
+        .unwrap_or_else(|| std::env::current_dir().expect("Failed to get current directory."));
+    let program_dir: Utf8PathBuf = program_dir
+        .try_into()
+        .expect("Failed to convert PathBuf to Utf8PathBuf");
+
+    // Get the program metadata.
+    let program_metadata_file = program_dir.join("Cargo.toml");
+    let mut program_metadata_cmd = cargo_metadata::MetadataCommand::new();
+    let program_metadata = program_metadata_cmd
+        .manifest_path(program_metadata_file)
+        .exec()?;
+
+    // Get the command corresponding to Docker or local build.
+    let cmd = create_local_command(args, &program_dir, &program_metadata);
+
+    execute_command(cmd)?;
+
+    copy_elf_to_output_dir(args, &program_metadata)
+}
+
+/// Internal helper function to build the program with or without arguments.
+pub(crate) fn build_program_internal(path: &str, args: Option<BuildArgs>) {
+    // Get the root package name and metadata.
+    let program_dir = std::path::Path::new(path);
+    let metadata_file = program_dir.join("Cargo.toml");
+    let mut metadata_cmd = cargo_metadata::MetadataCommand::new();
+    let metadata = metadata_cmd.manifest_path(metadata_file).exec().unwrap();
+    let root_package = metadata.root_package();
+    let root_package_name = root_package
+        .as_ref()
+        .map(|p| p.name.as_str())
+        .unwrap_or("Program");
+
+    // Skip the program build if the ZKM_SKIP_PROGRAM_BUILD environment variable is set to true.
+    let skip_program_build = std::env::var("ZKM_SKIP_PROGRAM_BUILD")
+        .map(|v| v.eq_ignore_ascii_case("true"))
+        .unwrap_or(false);
+    if skip_program_build {
+        println!(
+            "cargo:warning=Build skipped for {} at {} due to ZKM_SKIP_PROGRAM_BUILD flag",
+            root_package_name,
+            current_datetime()
+        );
+        return;
+    }
+
+    // Activate the build command if the dependencies change.
+    cargo_rerun_if_changed(&metadata, program_dir);
+
+    // Check if RUSTC_WORKSPACE_WRAPPER is set to clippy-driver (i.e. if `cargo clippy` is the
+    // current compiler). If so, don't execute `cargo prove build` because it breaks
+    // rust-analyzer's `cargo clippy` feature.
+    let is_clippy_driver = std::env::var("RUSTC_WORKSPACE_WRAPPER")
+        .map(|val| val.contains("clippy-driver"))
+        .unwrap_or(false);
+    if is_clippy_driver {
+        println!("cargo:warning=Skipping build due to clippy invocation.");
+        return;
+    }
+
+    // Build the program with the given arguments.
+    let path_output = if let Some(args) = args {
+        execute_build_program(&args, Some(program_dir.to_path_buf()))
+    } else {
+        execute_build_program(&BuildArgs::default(), Some(program_dir.to_path_buf()))
+    };
+    if let Err(err) = path_output {
+        panic!("Failed to build Zkm program: {}.", err);
+    }
+
+    println!(
+        "cargo:warning={} built at {}",
+        root_package_name,
+        current_datetime()
+    );
+}

build/src/command/local.rs

@@ -0,0 +1,39 @@
+use std::process::Command;
+
+use crate::{BuildArgs, HELPER_TARGET_SUBDIR};
+use cargo_metadata::camino::Utf8PathBuf;
+
+use super::utils::{get_program_build_args, get_rust_compiler_flags};
+
+/// Get the command to build the program locally.
+pub(crate) fn create_local_command(
+    args: &BuildArgs,
+    program_dir: &Utf8PathBuf,
+    program_metadata: &cargo_metadata::Metadata,
+) -> Command {
+    let mut command = Command::new("cargo");
+    let canonicalized_program_dir = program_dir
+        .canonicalize()
+        .expect("Failed to canonicalize program directory");
+
+    // When executing the local command:
+    // 1. Set the target directory to a subdirectory of the program's target directory to avoid
+    //    build
+    // conflicts with the parent process. Source: https://github.com/rust-lang/cargo/issues/6412
+    // 2. Set the rustup toolchain to succinct.
+    // 3. Set the encoded rust flags.
+    // 4. Remove the rustc configuration, otherwise in a build script it will attempt to compile the
+    //    program with the toolchain of the normal build process, rather than the Succinct
+    //    toolchain.
+    command
+        .current_dir(canonicalized_program_dir)
+        .env("RUSTUP_TOOLCHAIN", "nightly-2023-04-06")
+        .env("CARGO_ENCODED_RUSTFLAGS", get_rust_compiler_flags())
+        .env_remove("RUSTC")
+        .env(
+            "CARGO_TARGET_DIR",
+            program_metadata.target_directory.join(HELPER_TARGET_SUBDIR),
+        )
+        .args(get_program_build_args(args));
+    command
+}

build/src/command/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod local;
+pub(crate) mod utils;

build/src/command/utils.rs

@@ -0,0 +1,91 @@
+use anyhow::{Context, Result};
+use std::{
+    io::{BufRead, BufReader},
+    process::{exit, Command, Stdio},
+    thread,
+};
+
+use crate::{BuildArgs, BUILD_TARGET};
+
+/// Get the arguments to build the program with the arguments from the [`BuildArgs`] struct.
+pub(crate) fn get_program_build_args(args: &BuildArgs) -> Vec<String> {
+    let mut build_args = vec![
+        "build".to_string(),
+        "--release".to_string(),
+        "--target".to_string(),
+        BUILD_TARGET.to_string(),
+    ];
+
+    if args.ignore_rust_version {
+        build_args.push("--ignore-rust-version".to_string());
+    }
+
+    if !args.binary.is_empty() {
+        build_args.push("--bin".to_string());
+        build_args.push(args.binary.clone());
+    }
+
+    if !args.features.is_empty() {
+        build_args.push("--features".to_string());
+        build_args.push(args.features.join(","));
+    }
+
+    if args.no_default_features {
+        build_args.push("--no-default-features".to_string());
+    }
+
+    if args.locked {
+        build_args.push("--locked".to_string());
+    }
+
+    build_args
+}
+
+/// Rust flags for compilation of C libraries.
+pub(crate) fn get_rust_compiler_flags() -> String {
+    let rust_flags = [
+        "-C".to_string(),
+        "target-cpu=mips32".to_string(),
+        "--cfg".to_string(),
+        "target_os=\"zkvm\"".to_string(),
+        "-C".to_string(),
+        "target-feature=+crt-static".to_string(),
+        "-C".to_string(),
+        "link-arg=-g".to_string(),
+    ];
+    rust_flags.join("\x1f")
+}
+
+/// Execute the command and handle the output depending on the context.
+pub(crate) fn execute_command(mut command: Command) -> Result<()> {
+    // Add necessary tags for stdout and stderr from the command.
+    let mut child = command
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .context("failed to spawn command")?;
+    let stdout = BufReader::new(child.stdout.take().unwrap());
+    let stderr = BufReader::new(child.stderr.take().unwrap());
+
+    // Add prefix to the output of the process depending on the context.
+    let msg = "[zkm] ";
+
+    // Pipe stdout and stderr to the parent process with [docker] prefix
+    let stdout_handle = thread::spawn(move || {
+        stdout.lines().for_each(|line| {
+            println!("{} {}", msg, line.unwrap());
+        });
+    });
+    stderr.lines().for_each(|line| {
+        eprintln!("{} {}", msg, line.unwrap());
+    });
+    stdout_handle.join().unwrap();
+
+    // Wait for the child process to finish and check the result.
+    let result = child.wait()?;
+    if !result.success() {
+        // Error message is already printed by cargo.
+        exit(result.code().unwrap_or(1))
+    }
+    Ok(())
+}

build/src/lib.rs

@@ -0,0 +1,97 @@
+mod build;
+mod command;
+mod utils;
+use build::build_program_internal;
+pub use build::execute_build_program;
+
+use clap::Parser;
+
+const BUILD_TARGET: &str = "mips-unknown-linux-musl";
+// const DEFAULT_TAG: &str = "v1.0.0";
+const DEFAULT_OUTPUT_DIR: &str = "elf";
+const HELPER_TARGET_SUBDIR: &str = "elf-compilation";
+
+/// Compile an ZKM program.
+///
+/// Additional arguments are useful for configuring the build process, including options for using
+/// Docker, specifying binary and ELF names, ignoring Rust version checks, and enabling specific
+/// features.
+#[derive(Clone, Parser, Debug)]
+pub struct BuildArgs {
+    #[clap(
+        long,
+        action,
+        value_delimiter = ',',
+        help = "Space or comma separated list of features to activate"
+    )]
+    pub features: Vec<String>,
+    #[clap(long, action, help = "Do not activate the `default` feature")]
+    pub no_default_features: bool,
+    #[clap(long, action, help = "Ignore `rust-version` specification in packages")]
+    pub ignore_rust_version: bool,
+    #[clap(long, action, help = "Assert that `Cargo.lock` will remain unchanged")]
+    pub locked: bool,
+    #[clap(
+        alias = "bin",
+        long,
+        action,
+        help = "Build only the specified binary",
+        default_value = ""
+    )]
+    pub binary: String,
+    #[clap(long, action, help = "ELF binary name", default_value = "")]
+    pub elf_name: String,
+    #[clap(
+        alias = "out-dir",
+        long,
+        action,
+        help = "Copy the compiled ELF to this directory",
+        default_value = DEFAULT_OUTPUT_DIR
+    )]
+    pub output_directory: String,
+}
+
+// Implement default args to match clap defaults.
+impl Default for BuildArgs {
+    fn default() -> Self {
+        Self {
+            features: vec![],
+            ignore_rust_version: false,
+            binary: "".to_string(),
+            elf_name: "".to_string(),
+            output_directory: DEFAULT_OUTPUT_DIR.to_string(),
+            locked: false,
+            no_default_features: false,
+        }
+    }
+}
+
+/// Builds the program if the program at the specified path, or one of its dependencies, changes.
+///
+/// This function monitors the program and its dependencies for changes. If any changes are
+/// detected, it triggers a rebuild of the program.
+///
+/// # Arguments
+///
+/// * `path` - A string slice that holds the path to the program directory.
+///
+/// This function is useful for automatically rebuilding the program during development
+/// when changes are made to the source code or its dependencies.
+///
+/// Set the `ZKM_SKIP_PROGRAM_BUILD` environment variable to `true` to skip building the program.
+pub fn build_program(path: &str) {
+    build_program_internal(path, None)
+}
+
+/// Builds the program with the given arguments if the program at path, or one of its dependencies,
+/// changes.
+///
+/// # Arguments
+///
+/// * `path` - A string slice that holds the path to the program directory.
+/// * `args` - A [`BuildArgs`] struct that contains various build configuration options.
+///
+/// Set the `ZKM_SKIP_PROGRAM_BUILD` environment variable to `true` to skip building the program.
+pub fn build_program_with_args(path: &str, args: BuildArgs) {
+    build_program_internal(path, Some(args))
+}