feat: impl basic test suite loader and runner

Signed-off-by: peefy <xpf6677@163.com>

kclvm/compiler/src/codegen/llvm/node.rs

@@ -261,7 +261,7 @@ impl<'ctx> TypedResultWalker<'ctx> for LLVMCodeGenContext<'ctx> {
         );
         self.br(end_block);
         self.builder.position_at_end(end_block);
-        self.ok_result()
+        Ok(self.undefined_value())
     }
 
     fn walk_if_stmt(&self, if_stmt: &'ctx ast::IfStmt) -> Self::Result {

kclvm/driver/src/lib.rs

@@ -9,14 +9,15 @@ mod tests;
 use glob::glob;
 use kclvm_ast::ast;
 use kclvm_config::{
-    modfile::{KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV},
+    modfile::{get_pkg_root, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV},
     path::ModRelativePath,
     settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE},
 };
 use kclvm_parser::LoadProgramOptions;
 use kclvm_utils::path::PathPrefix;
 use kpm_metadata::fill_pkg_maps_for_k_file;
 use std::{
+    collections::HashSet,
     fs::read_dir,
     io::{self, ErrorKind},
     path::{Path, PathBuf},
@@ -286,3 +287,84 @@ pub fn get_kcl_files<P: AsRef<Path>>(path: P, recursively: bool) -> Result<Vec<S
     files.sort();
     Ok(files)
 }
+
+/// Get the package string list form the package path.
+pub fn get_pkg_list(pkgpath: &str) -> Result<Vec<String>> {
+    let mut dir_list: Vec<String> = Vec::new();
+    let mut dir_map: HashSet<String> = HashSet::new();
+    let cwd = std::env::current_dir()?;
+
+    let pkgpath = if pkgpath.is_empty() {
+        cwd.to_string_lossy().to_string()
+    } else {
+        pkgpath.to_string()
+    };
+
+    let include_sub_pkg = pkgpath.ends_with("/...");
+    let pkgpath = if include_sub_pkg {
+        pkgpath.trim_end_matches("/...").to_string()
+    } else {
+        pkgpath
+    };
+
+    if pkgpath != "." && pkgpath.ends_with(".") {
+        return Ok(Vec::new());
+    }
+
+    if pkgpath.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    match pkgpath.chars().next() {
+        Some('.') => {
+            let pkgpath = Path::new(&cwd).join(&pkgpath);
+            pkgpath.to_string_lossy().to_string()
+        }
+        _ => {
+            if Path::new(&pkgpath).is_absolute() {
+                pkgpath.clone()
+            } else {
+                if !pkgpath.contains('/') && !pkgpath.contains('\\') {
+                    pkgpath.replace(".", "/")
+                } else {
+                    let pkgroot =
+                        get_pkg_root(&cwd.to_str().ok_or(anyhow::anyhow!("cwd path not found"))?)
+                            .unwrap_or_default();
+                    if !pkgroot.is_empty() {
+                        PathBuf::from(pkgroot)
+                            .join(&pkgpath)
+                            .to_string_lossy()
+                            .to_string()
+                    } else {
+                        Path::new(&cwd).join(&pkgpath).to_string_lossy().to_string()
+                    }
+                }
+            }
+        }
+    };
+
+    if !include_sub_pkg {
+        return Ok(vec![pkgpath]);
+    }
+
+    for entry in WalkDir::new(&pkgpath).into_iter().filter_map(|e| e.ok()) {
+        let path = entry.path();
+        if !path.is_dir() {
+            if path.extension().and_then(|ext| ext.to_str()) == Some(KCL_FILE_EXTENSION)
+                && !path
+                    .file_name()
+                    .map(|name| name.to_string_lossy().starts_with("_"))
+                    .unwrap_or(false)
+            {
+                if let Some(dir) = path.parent().map(|p| p.to_string_lossy().to_string()) {
+                    if !dir_map.contains(&dir) {
+                        dir_list.push(dir.clone());
+                        dir_map.insert(dir);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(dir_list)
+}

kclvm/runner/src/lib.rs

@@ -7,15 +7,14 @@ use kclvm_ast::{
     MAIN_PKG,
 };
 use kclvm_driver::{canonicalize_input_files, expand_input_files};
-use kclvm_error::{Diagnostic, Handler};
 use kclvm_parser::{load_program, ParseSession};
 use kclvm_query::apply_overrides;
-use kclvm_runtime::{Context, PanicInfo, PlanOptions, ValueRef};
+use kclvm_runtime::{Context, PlanOptions, ValueRef};
 use kclvm_sema::resolver::{
     resolve_program, resolve_program_with_opts, scope::ProgramScope, Options,
 };
 use linker::Command;
-pub use runner::{ExecProgramArgs, ExecProgramResult, MapErrorResult};
+pub use runner::{Artifact, ExecProgramArgs, ExecProgramResult, MapErrorResult};
 use runner::{KclLibRunner, KclLibRunnerOptions};
 use tempfile::tempdir;
 
@@ -74,14 +73,8 @@ pub mod tests;
 pub fn exec_program(sess: Arc<ParseSession>, args: &ExecProgramArgs) -> Result<ExecProgramResult> {
     // parse args from json string
     let opts = args.get_load_program_options();
-    let k_files = &args.k_filename_list;
-    let work_dir = args.work_dir.clone().unwrap_or_default();
-    let k_files = expand_input_files(k_files);
-    let kcl_paths =
-        canonicalize_input_files(&k_files, work_dir, false).map_err(|err| anyhow!(err))?;
-
+    let kcl_paths = expand_files(args)?;
     let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
-
     let mut program = load_program(sess.clone(), kcl_paths_str.as_slice(), Some(opts), None)
         .map_err(|err| anyhow!(err))?;
 
@@ -212,22 +205,10 @@ pub fn execute(
     let runner = KclLibRunner::new(Some(KclLibRunnerOptions {
         plugin_agent_ptr: args.plugin_agent,
     }));
-    let mut result = runner.run(&lib_path, args)?;
+    let result = runner.run(&lib_path, args)?;
 
     remove_file(&lib_path)?;
     clean_tmp_files(&temp_entry_file, &lib_suffix)?;
-    // Wrap runtime error into diagnostic style string.
-    if !result.err_message.is_empty() {
-        result.err_message = match Handler::default()
-            .add_diagnostic(<PanicInfo as Into<Diagnostic>>::into(PanicInfo::from(
-                result.err_message.as_str(),
-            )))
-            .emit_to_string()
-        {
-            Ok(msg) => msg,
-            Err(err) => err.to_string(),
-        };
-    }
     Ok(result)
 }
 
@@ -256,6 +237,64 @@ pub fn execute_module(mut m: Module) -> Result<ExecProgramResult> {
     )
 }
 
+///
+pub fn build_program<P: AsRef<Path>>(
+    sess: Arc<ParseSession>,
+    args: &ExecProgramArgs,
+    output: Option<P>,
+) -> Result<Artifact> {
+    // Parse program.
+    let opts = args.get_load_program_options();
+    let kcl_paths = expand_files(args)?;
+    let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
+    let mut program = load_program(sess.clone(), kcl_paths_str.as_slice(), Some(opts), None)
+        .map_err(|err| anyhow!(err))?;
+    // Resolve program.
+    let scope = resolve_program(&mut program);
+    emit_compile_diag_to_string(sess, &scope, false)?;
+    // Create a temp entry file and the temp dir will be delete automatically.
+    let temp_dir = tempdir()?;
+    let temp_dir_path = temp_dir.path().to_str().ok_or(anyhow!(
+        "Internal error: {}: No such file or directory",
+        temp_dir.path().display()
+    ))?;
+    let temp_entry_file = temp_file(temp_dir_path)?;
+    // Generate native libs.
+    let lib_paths = assembler::KclvmAssembler::new(
+        program,
+        scope,
+        temp_entry_file.clone(),
+        KclvmLibAssembler::LLVM,
+        args.get_package_maps_from_external_pkg(),
+    )
+    .gen_libs()?;
+    // Link libs into one library.
+    let lib_suffix = Command::get_lib_suffix();
+    let temp_out_lib_file = if let Some(output) = output {
+        let path = output
+            .as_ref()
+            .to_str()
+            .ok_or(anyhow!("build output path is not found"))?
+            .to_string();
+        path
+    } else {
+        format!("{}{}", temp_entry_file, lib_suffix)
+    };
+    let lib_path = linker::KclvmLinker::link_all_libs(lib_paths, temp_out_lib_file)?;
+    // Return the library artifact.
+    Artifact::load_from_path(lib_path)
+}
+
+/// Expand and return the normalized file paths for the input file list.
+pub fn expand_files(args: &ExecProgramArgs) -> Result<Vec<String>> {
+    let k_files = &args.k_filename_list;
+    let work_dir = args.work_dir.clone().unwrap_or_default();
+    let k_files = expand_input_files(k_files);
+    let kcl_paths =
+        canonicalize_input_files(&k_files, work_dir, false).map_err(|err| anyhow!(err))?;
+    Ok(kcl_paths)
+}
+
 /// Clean all the tmp files generated during lib generating and linking.
 #[inline]
 fn clean_tmp_files(temp_entry_file: &String, lib_suffix: &String) -> Result<()> {

kclvm/runner/src/runner.rs

@@ -6,9 +6,11 @@ use kclvm_config::{
     modfile::get_vendor_home,
     settings::{SettingsFile, SettingsPathBuf},
 };
+use kclvm_error::{Diagnostic, Handler};
 use kclvm_query::r#override::parse_override_spec;
-use kclvm_runtime::{Context, ValueRef};
+use kclvm_runtime::{Context, PanicInfo, ValueRef};
 use serde::{Deserialize, Serialize};
+use std::ffi::OsStr;
 
 const RESULT_SIZE: usize = 2048 * 2048;
 
@@ -206,6 +208,28 @@ impl TryFrom<SettingsPathBuf> for ExecProgramArgs {
     }
 }
 
+pub struct Artifact(libloading::Library);
+
+pub trait ProgramRun {
+    fn run(&self, args: &ExecProgramArgs) -> Result<ExecProgramResult>;
+}
+
+impl ProgramRun for Artifact {
+    fn run(&self, args: &ExecProgramArgs) -> Result<ExecProgramResult> {
+        unsafe {
+            KclLibRunner::lib_kclvm_plugin_init(&self.0, args.plugin_agent)?;
+            KclLibRunner::lib_kcl_run(&self.0, args)
+        }
+    }
+}
+
+impl Artifact {
+    pub fn load_from_path<P: AsRef<OsStr>>(path: P) -> Result<Self> {
+        let lib = unsafe { libloading::Library::new(path)? };
+        Ok(Self(lib))
+    }
+}
+
 #[derive(Debug, Default)]
 pub struct KclLibRunnerOptions {
     pub plugin_agent_ptr: u64,
@@ -379,6 +403,20 @@ impl KclLibRunner {
             let return_len = 0 - n;
             result.err_message = String::from_utf8(warn_data[0..return_len as usize].to_vec())?;
         }
+
+        // Wrap runtime error into diagnostic style string.
+        if !result.err_message.is_empty() {
+            result.err_message = match Handler::default()
+                .add_diagnostic(<PanicInfo as Into<Diagnostic>>::into(PanicInfo::from(
+                    result.err_message.as_str(),
+                )))
+                .emit_to_string()
+            {
+                Ok(msg) => msg,
+                Err(err) => err.to_string(),
+            };
+        }
+
         Ok(result)
     }
 }

kclvm/sema/src/resolver/node.rs

@@ -978,7 +978,10 @@ impl<'ctx> MutSelfTypedResultWalker<'ctx> for Resolver<'ctx> {
         if let Some(stmt) = lambda_expr.body.last() {
             if !matches!(
                 stmt.node,
-                ast::Stmt::Expr(_) | ast::Stmt::Assign(_) | ast::Stmt::AugAssign(_)
+                ast::Stmt::Expr(_)
+                    | ast::Stmt::Assign(_)
+                    | ast::Stmt::AugAssign(_)
+                    | ast::Stmt::Assert(_)
             ) {
                 self.handler.add_compile_error(
                     "The last statement of the lambda body must be a expression e.g., x, 1, etc.",

kclvm/tools/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod fix;
 pub mod format;
 pub mod lint;
+pub mod testing;
 pub mod util;
 pub mod vet;

kclvm/tools/src/testing/mod.rs

@@ -0,0 +1,44 @@
+//! [kclvm_tools::test] module mainly contains some functions of language testing tool.
+pub use crate::testing::suite::{load_test_suites, TestSuite};
+use anyhow::{Error, Result};
+use indexmap::IndexMap;
+use kclvm_runner::ExecProgramArgs;
+use std::time::Duration;
+
+mod suite;
+
+#[cfg(test)]
+mod tests;
+
+pub trait TestRun {
+    type Options;
+    type Result;
+    fn run(&self, opts: &Self::Options) -> Result<Self::Result>;
+}
+
+pub trait TestRunner {
+    type Suite: TestRun;
+    type Result;
+    type Options;
+    fn look_up_test_suites(&self, path: &str, opts: &Self::Options) -> Result<Vec<Self::Suite>>;
+    fn run(&self, suites: &[Self::Suite], opts: &Self::Options) -> Vec<Result<Self::Result>>;
+}
+
+#[derive(Debug, Default)]
+pub struct TestResult {
+    pub log_message: String,
+    pub info: IndexMap<String, TestCaseInfo>,
+}
+
+#[derive(Debug, Default)]
+pub struct TestCaseInfo {
+    pub error: Option<Error>,
+    pub duration: Duration,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TestOptions {
+    pub exec_args: ExecProgramArgs,
+    pub run_regexp: String,
+    pub fail_fast: bool,
+}