tests: rewrite (xeet based)

Kill the old internal testing tools in favor of XEET. Also modifies the
way packages are tests (though the jist of it remains the same).

Signed-off-by: Omer Caspi <omer.caspi@gmail.com>

.github/scripts/run_test.sh

@@ -4,6 +4,10 @@
 
 OS=$(uname -s)
 
-opts="--no-colors"
-[[ ${OS} != "Linux" ]] && opts+=" --no-containers"
-tests/tr_test --no-colors ${opts}
+opts=""
+[[ ${OS} != "Linux" ]] && opts+="-X container"
+
+export USE_VENV=0
+cd ${GITHUB_WORKSPACE}/tests || exit 1
+
+xeet --no-colors run -c xeet.json ${opts}

.github/scripts/setup.sh

@@ -7,11 +7,10 @@ echo ${OS}
 [[ ${OS} == "Darwin" ]] && brew install coreutils
 
 python -m pip install --upgrade pip
-pip install -r requirements.txt
 if [[ ${OS} == "Linux" ]]; then
 	podman pull docker.io/rockylinux/rockylinux:9.3
 	podman pull docker.io/library/ubuntu:24.04
 fi
 
-pip install -e .
+pip install -e .[devel]
 

.gitignore

@@ -3,8 +3,7 @@ pyrightconfig.json
 .vimspector.json
 .idea/
 __pycache__/
-tests/output
-tests/intermediate.expected
+tests/xeet.out/
 
 # Virtual Environments
 .env/

.tasks.json

@@ -31,25 +31,22 @@
 		},
 		"test_pkg_base": {
 			"short_desc": "Basic package test",
-			"cwd": "{{taskRoot}}/tests",
 			"commands": [
-				"./tr_test"
+				"{{taskRoot}}/tests/run_pkg_tests"
 			],
 			"abstract": true
 		},
 		"test_pkg_testpypi": {
 			"short_desc": "Basic package test from testpypi",
 			"base": "test_pkg_base",
 			"env": {
-				"TEST_PKG": "1",
 				"TEST_PYPI": "1"
 			}
 		},
 		"test_pkg_pypi": {
 			"short_desc": "Basic package test from pypi",
 			"base": "test_pkg_base",
 			"env": {
-				"TEST_PKG": "1",
 				"TEST_PYPI": "0"
 			}
 		}

pyproject.toml

@@ -27,6 +27,11 @@ dependencies = [
 ]
 requires-python = ">=3.10,<4"
 
+[project.optional-dependencies]
+devel = [
+	"xeet>=0.5.5"
+]
+
 [project.urls]
 "Homepage" = "https://github.com/omercsp/taskrunner"
 

tests/base_tasks_a.json

@@ -8,10 +8,7 @@
 		"042_included": {
 			"commands": [
 				"echo base a"
-			],
-			"meta": {
-				"comment": "should be overridden by a task in the base_tasks_b.json, and then again by main tasks file"
-			}
+			]
 		}
 	},
 	"use_default_include": false

tests/base_tasks_b.json

@@ -10,10 +10,7 @@
 		"041_included": {
 			"commands": [
 				"echo base b"
-			],
-			"meta": {
-				"comment": "should be overridden by a task in the maing tasks file"
-			}
+			]
 		}
 	},
 	"use_default_include": false

tests/diff_test_output.sh

@@ -0,0 +1 @@
+scripts/diff_test_output.sh

tests/aux/tasks-for-list.json → tests/for_list_test/tasks-for-list.json

@@ -31,7 +31,7 @@
 			"base": "no-such-task",
 			"short_desc": "This will not be displayed"
 		},
-		"a_very_long_task_name_with_lots_of_words_and whatever": {
+		"a_very_long_task_name_with_lots_of_words_and_whatever": {
 			"short_desc": "Even longer description string with lots and lots of words"
 		}
 	},

tests/tr_test → tests/run_pkg_tests

@@ -1,8 +1,8 @@
 #!/bin/bash
+set -e
 
 TEST_PKG=${TEST_PKG:-0}
 TEST_PYPI=${TEST_PYPI:-0}
-TR_TEST=tr_test.py
 
 # Stupid Apple
 if [[ ${OSTYPE} == 'darwin'* ]]; then
@@ -17,7 +17,7 @@ venv_dir=""
 
 _abort()
 {
-	echo $*
+	echo "$@"
 	echo
 	exit 1
 }
@@ -37,9 +37,10 @@ _build_pkg_venv()
 	echo "Creating venv at ${venv_dir}"
 	python -m venv ${venv_dir}
 	source ${venv_dir}/bin/activate
+	pip install --upgrade pip
 
-	pip install ${repo_flags} pytaskrunner
-	which task
+	# shellcheck disable=SC2102
+	pip install ${repo_flags} pytaskrunner[devel]
 }
 
 _fini_pkg_env()
@@ -48,12 +49,16 @@ _fini_pkg_env()
     [[ -n ${venv_dir} && -d ${venv_dir} ]] && rm -rf ${venv_dir}
 }
 
-cd ${SCRIPT_DIR}
-[[ ! -e ${TR_TEST} ]] && _abort  "'${TR_TEST} can't be found in current directory"
-[[ ${TEST_PKG} -eq 1 ]] && _build_pkg_venv
+_build_pkg_venv
+
+echo "============================"
+echo "Taskrunner tests direcotry: ${SCRIPT_DIR}"
+echo "VENV directory: ${venv_dir}"
+echo "xeet binary: $(which xeet)"
+echo "============================"
+xeet run -c ${SCRIPT_DIR}/xeet.json "$@"
 
-python3 ${TR_TEST} "$@"
 ret=$?
-[[ ${TEST_PKG} -eq 1 ]] && _fini_pkg_env
+_fini_pkg_env
 
 exit ${ret}

tests/runtests

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+_abort()
+{
+	echo "ERROR: $1"
+	exit 1
+}
+
+PROJ_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
+[[ -z "${PROJ_ROOT}" ]] && _abort "Could not find project root. Are you in the project directory?"
+
+HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd ${HERE} || _abort "Could not cd to ${HERE}"
+
+if [[ -z "${VENV_ACTIVATE}" ]]; then
+	source ${PROJ_ROOT}/.venv/bin/activate || _abort "ERROR: Could not activate virtualenv"
+fi
+
+cmd="xeet run -c xeet.json $*"
+echo "Running: ${cmd}"
+${cmd}

tests/scripts/common.inc.sh

@@ -0,0 +1,105 @@
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd ${DIR} || exit 1
+
+echoerr()
+{
+    echo "$@" 1>&2;
+}
+
+require_var()
+{
+    local var_name=$1
+    #make sure the variable is set
+    if [[ -z ${!var_name} ]]; then
+        echoerr "Missing variable ${var_name}"
+        exit 1
+    fi
+}
+
+require_dir()
+{
+    local dir_name=$1
+    #make sure the variable is set
+    if [[ ! -d ${dir_name} ]]; then
+        echoerr "Missing directory ${dir_name}"
+        exit 1
+    fi
+}
+
+require_dir_var()
+{
+    local dir_var_name=$1
+    require_var ${dir_var_name}
+    require_dir ${!dir_var_name}
+}
+
+require_file()
+{
+    local file_name=$1
+    local file_name_pr=${file_name#"${XEET_ROOT}/"}
+    #make sure the variable is set
+    if [[ ! -f ${file_name} ]]; then
+        echoerr "Missing file: ${file_name_pr}"
+        exit 1
+    fi
+}
+
+require_not_file()
+{
+    local file_name=$1
+    #make sure the variable is set
+    if [[ -f ${file_name} ]]; then
+        echoerr "File ${file_name} should not exist"
+        exit 1
+    fi
+}
+
+require_var XEET_TEST_NAME
+require_var XEET_TEST_DEBUG
+
+require_dir_var XEET_TEST_OUTPUT_DIR
+require_dir_var XEET_ROOT
+
+TEST_EXPECTED_DIR=${XEET_ROOT}/xeet.expected/${XEET_TEST_NAME}
+
+_compare_file()
+{
+    local expected_file=$1
+    local out_file=$2
+    local expected_file_pr=${expected_file#"${XEET_ROOT}/"}
+    local out_file_pr=${out_file#"${XEET_ROOT}/"}
+
+    if [[ -f ${expected_file} ]]; then
+        require_file ${out_file}
+    else
+        if [[ ! -f ${out_file} ]]; then
+            return 0
+        fi
+    fi
+
+    if [[ -f ${out_file} ]]; then
+        require_file ${expected_file}
+    else
+        require_not_file ${expected_file}
+    fi
+
+    if diff -q ${out_file} ${expected_file} &> /dev/null; then
+        echo "File ${out_file_pr} matches expected output"
+    else
+        echoerr "Files ${out_file_pr} and ${expected_file_pr} do not match"
+	echoerr "Diff saved to ${out_file_pr}.diff"
+        diff -u ${out_file} ${expected_file} &> ${out_file}.diff
+        return 1
+    fi
+}
+
+generic_stdout_test()
+{
+    if [[ ${XEET_TEST_DEBUG} == "1" ]]; then
+        echo "Running in debug mode, skipping stdout comparison"
+        return 0
+    fi
+    _compare_file ${TEST_EXPECTED_DIR}/stdout ${XEET_TEST_OUTPUT_DIR}/stdout
+    _compare_file ${TEST_EXPECTED_DIR}/stderr ${XEET_TEST_OUTPUT_DIR}/stderr
+}
+

tests/scripts/devel_common.inc.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# shellcheck disable=SC2034
+EXPECTED_BASE_DIR=xeet.expected
+OUT_BASE_DIR=xeet.out
+OUTPUT_FILES="stdout stderr"
+
+echoerr()
+{
+    echo "$@" 1>&2;
+}
+warn()
+{
+	local args="$*"
+
+	# print warning in yellow
+	echo -e "\033[33m${args}033[0m"
+}
+abort()
+{
+	echoerr Aborting: "$*"
+	exit 1
+}
+
+_get_test_dir()
+{
+	local base_dir=$1
+	local test_name=$2
+
+	if [[ -d ${base_dir}/${test_name} ]]; then
+		echo ${base_dir}/${test_name}
+	else
+		readarray -t test_directories < <(find ${base_dir} -name "${test_name}*" -type d -printf "%f\n")
+		if [[ ${#test_directories[@]} -eq 0 ]]; then
+			echo "No directory found for test ${test_name} in ${base_dir}"
+			return 1
+		fi
+
+		if [[ ${#test_directories[@]} -gt 1 ]]; then
+			echo "Multiple directories found for test ${test_name} in ${base_dir}"
+			return 1
+		fi
+		echo ${base_dir}/${test_directories[0]}
+	fi
+}

tests/scripts/dflt_output_compare.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source ${DIR}/common.inc.sh
+
+generic_stdout_test