run-regtests.sh

#!/bin/sh

_help()
{
  cat << EOF
### run-regtests.sh ###
  Running run-regtests.sh --help shows this information about how to use it

  Run without parameters to run all tests in the current folder (including subfolders)
    run-regtests.sh

  Provide paths to run tests from (including subfolders):
    run-regtests.sh ./tests1 ./tests2

  Parameters:
    --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
      run-regtests.sh ./fasttest --j 16

    --v, to run verbose
      run-regtests.sh --v, disables the default vtest 'quiet' parameter

    --debug to show test logs on standard output (implies --v)
      run-regtests.sh --debug

    --keep-logs to keep all log directories (by default kept if test fails)
      run-regtests.sh --keep-logs

    --vtestparams <ARGS>, passes custom ARGS to vtest
      run-regtests.sh --vtestparams "-n 10"

    --type <reg tests types> filter the types of the tests to be run, depending on
      the commented REGTESTS_TYPE variable value in each VTC file.
      The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
      and experimental. When not specified, it is set to 'default' as default value.

      run-regtest.sh --type slow,default

    --clean to cleanup previous reg-tests log directories and exit
      run-regtests.sh --clean

  Including text below into a .vtc file will check for its requirements
  related to haproxy's target and compilation options
    # Below targets are not capable of completing this test successfully
    #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd

    #EXCLUDE_TARGETS=dos,freebsd,windows

    # Below option is required to complete this test successfully
    #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
    #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA

    #REQUIRE_SERVICE=prometheus-exporter
    #REQUIRE_SERVICES=prometheus-exporter,foo

    # To define a range of versions that a test can run with:
    #REQUIRE_VERSION=0.0
    #REQUIRE_VERSION_BELOW=99.9

  Configure environment variables to set the haproxy and vtest binaries to use
    setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
    setenv VTEST_PROGRAM /usr/local/bin/vtest
    setenv HAPROXY_ARGS "-dM -de -m 50"
  or
    export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
    export VTEST_PROGRAM=/usr/local/bin/vtest
    export HAPROXY_ARGS="-dM -de -m 50"
EOF
  exit 0
}

add_range_to_test_list()
{
    level0="*.vtc"
    level1="h*.vtc"
    level2="s*.vtc"
    level3="l*.vtc"
    level4="b*.vtc"
    level5="k*.vtc"
    level6="e*.vtc"

    new_range=$(echo $1 | tr '-' ' ')
    non_digit=$(echo $new_range | grep '[^0-9 ]')
    if [ -n "$non_digit" ] ; then
        return
    fi
    if [ "$new_range" = "$1" ] ; then
        if [ $1 -gt 6 ] ; then
            return
        fi
        eval echo '$'level$1
        return
    fi
    if [ -z "$new_range" ] ; then
        return
    fi
    list=
    for l in $(seq $new_range) ; do
        if [ -n "l" ] ; then
            if [ -z "$list" ] ; then
                list="$(eval echo '$'level${l})"
            else
                list="$list $(eval echo '$'level${l})"
            fi
        fi
    done

    echo $list
}

_startswith() {
  _str="$1"
  _sub="$2"
  echo "$_str" | grep "^$_sub" >/dev/null 2>&1
}

_findtests() {
  set -f

  REGTESTS_TYPES="${REGTESTS_TYPES:-default,bug,devel,slow}"
  any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
  for i in $( find "$1" -name *.vtc ); do
    skiptest=
    OLDIFS="$IFS"; IFS="$LINEFEED"
    set -- $(grep '^#[0-9A-Z_]*=' "$i")
    IFS="$OLDIFS"

    require_version=""; require_version_below=""; require_options="";
    require_services=""; exclude_targets=""; regtest_type=""
    requiredoption=""; requiredservice=""; excludedtarget="";

    while [ $# -gt 0 ]; do
      v="$1"; v="${v#*=}"
      case "$1" in
        "#REQUIRE_VERSION="*)       require_version="$v" ;;
        "#REQUIRE_VERSION_BELOW="*) require_version_below="$v" ;;
        "#REQUIRE_OPTIONS="*)       require_options="$v" ;;
        "#REQUIRE_SERVICES="*)      require_services="$v" ;;
        "#EXCLUDE_TARGETS="*)       exclude_targets="$v" ;;
        "#REGTEST_TYPE="*)          regtest_type="$v" ;;
        "#REQUIRE_OPTION="*)        requiredoption="${v%,*}" ;;
        "#REQUIRE_SERVICE="*)       required_service="${v%,*}" ;;
        "#EXCLUDE_TARGET="*)        excludedtarget="${v%,*}" ;;
        # Note: any new variable declared here must be initialized above.
      esac
      shift
    done

    if [ $any_test -ne 1 ] ; then
        if [ -z $regtest_type ] ; then
            regtest_type=default
        fi
        if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
            echo "  Skip $i because its type '"$regtest_type"' is excluded"
            skiptest=1
        fi
    fi

    if [ -n "$requiredoption" ]; then
      require_options="$require_options,$requiredoption"
    fi

    if [ -n "$requiredservice" ]; then
      require_services="$require_services,$requiredservice"
    fi

    if [ -n "$excludedtarget" ]; then
      exclude_targets="$exclude_targets,$excludedtarget"
    fi

    IFS=","; set -- $require_options;  IFS=$OLDIFS; require_options="$*"
    IFS=","; set -- $require_services; IFS=$OLDIFS; require_services="$*"
    IFS=","; set -- $exclude_targets;  IFS=$OLDIFS; exclude_targets="$*"

    if [ -n "$require_version" ]; then
      if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
        echo "  Skip $i because option haproxy is version: $HAPROXY_VERSION"
        echo "    REASON: this test requires at least version: $require_version"
        skiptest=1
      fi
    fi
    if [ -n "$require_version_below" ]; then
      if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
        echo "  Skip $i because option haproxy is version: $HAPROXY_VERSION"
        echo "    REASON: this test requires a version below: $require_version_below"
        skiptest=1
      fi
    fi

    for excludedtarget in $exclude_targets; do
      if [ "$excludedtarget" = "$TARGET" ]; then
        echo "  Skip $i because haproxy is compiled for the excluded target $TARGET"
        skiptest=1
      fi
    done

    for requiredoption in $require_options; do
      IFS="|"; set -- $requiredoption;  IFS=$OLDIFS; alternatives="$*"
      found=
      for alt in $alternatives; do
        if [ -z "${FEATURES_PATTERN##* +$alt *}" ]; then
          found=1;
	fi
      done
      if [ -z $found ]; then
        echo "  Skip $i because haproxy is not compiled with the required option $requiredoption"
        skiptest=1
      fi
    done

    for requiredservice in $require_services; do
      IFS="|"; set -- $requiredservice;  IFS=$OLDIFS; alternatives="$*"
      found=
      for alt in $alternatives; do
        if [ -z "${SERVICES_PATTERN##* $alt *}" ]; then
          found=1;
	fi
      done
      if [ -z $found ]; then
        echo "  Skip $i because haproxy is not compiled with the required service $requiredservice"
        skiptest=1
      fi
    done

    if [ -z $skiptest ]; then
      echo "  Add test: $i"
      testlist="$testlist $i"
    fi
  done
}

_cleanup()
{
  DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
  if [ -z "${DIRS}" ]; then
    echo "No reg-tests log directory found"
  else
    echo "Cleanup following reg-tests log directories:"
    for d in ${DIRS}; do
      echo  "    o ${TESTDIR}/$d"
    done
    read -p "Continue (y/n)?" reply
    case "$reply" in
      y|Y)
        for d in ${DIRS}; do
          rm -r "${TESTDIR}/$d"
        done
        echo "done"
        exit 0
        ;;
       *)
        echo "aborted"
        exit 1
        ;;
    esac
  fi
}


_process() {
  while [ ${#} -gt 0 ]; do
    if _startswith "$1" "-"; then
      case "${1}" in
        --j)
          jobcount="$2"
          shift
          ;;
        --vtestparams)
          vtestparams="$2"
          shift
          ;;
        --v)
          verbose=""
          ;;
        --debug)
          verbose=""
          debug="-v"
          ;;
        --keep-logs)
          keep_logs="-L"
          ;;
        --type)
	      REGTESTS_TYPES="$2"
	      shift
	      ;;
        --clean)
          _cleanup
          exit 0
          ;;
        --help)
          _help
          ;;
        *)
          echo "Unknown parameter : $1"
          exit 1
          ;;
      esac
    else
      REGTESTS="${REGTESTS} $1"
    fi
    shift 1
  done
}

# compute a version from up to 4 sub-version components, each multiplied
# by a power of 1000, and padded left with 0, 1 or 2 zeroes.
_version() {
  OLDIFS="$IFS"; IFS="."; set -- $*; IFS="$OLDIFS"
  set -- ${1%%[!0-9]*} 000${2%%[!0-9]*} 000${3%%[!0-9]*} 000${4%%[!0-9]*}
  prf2=${2%???}; prf3=${3%???}; prf4=${4%???}
  echo ${1}${2#$prf2}${3#$prf3}${4#$prf4}
}


HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
HAPROXY_ARGS="${HAPROXY_ARGS--dM}"
VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
TESTDIR="${TMPDIR:-/tmp}"
REGTESTS=""
LINEFEED="
"

jobcount=""
verbose="-q"
debug=""
keep_logs="-l"
testlist=""

_process "$@";

echo ""
echo "########################## Preparing to run tests ##########################"

preparefailed=
if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
  echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
  preparefailed=1
fi
if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
  echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
  preparefailed=1
fi
if [ $preparefailed ]; then
  exit 1
fi

{ read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF
$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep 'HA-\?Proxy version\|TARGET.*=\|^Feature\|^Available services' | sed 's/.* [:=] //')
EOF

HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
echo "Testing with haproxy version: $HAPROXY_VERSION"

PROJECT_VERSION=$(${MAKE:-make} version 2>&1 | grep '^VERSION:\|^SUBVERS:'|cut -f2 -d' '|tr -d '\012')
if [ -z "${PROJECT_VERSION}${MAKE}" ]; then
        # try again with gmake, just in case
        PROJECT_VERSION=$(gmake version 2>&1 | grep '^VERSION:\|^SUBVERS:'|cut -f2 -d' '|tr -d '\012')
fi

FEATURES_PATTERN=" $FEATURES "
SERVICES_PATTERN=" $SERVICES "

TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"

mkdir -p "$TESTDIR" || exit 1
TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1

export TMPDIR="$TESTDIR"
export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
if [ -n "$HAPROXY_ARGS" ]; then
   export HAPROXY_ARGS
fi

echo "Target : $TARGET"
echo "Options : $FEATURES"
echo "Services : $SERVICES"

echo "########################## Gathering tests to run ##########################"

if [ -z "$REGTESTS" ]; then
  _findtests reg-tests/
else
  for t in $REGTESTS; do
    _findtests $t
  done
fi

echo "########################## Starting vtest ##########################"
echo "Testing with haproxy version: $HAPROXY_VERSION"

if [ -n "$PROJECT_VERSION" -a "$PROJECT_VERSION" != "$HAPROXY_VERSION" ]; then
        echo "Warning: version does not match the current tree ($PROJECT_VERSION)"
fi

_vtresult=0
if [ -n "$testlist" ]; then
  if [ -n "$jobcount" ]; then
    jobcount="-j $jobcount"
  fi
  cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
  eval $cmd
  _vtresult=$?
else
  echo "No tests found that meet the required criteria"
fi


if [ $_vtresult -eq 0 ]; then
  # all tests were successful, removing tempdir (the last part.)
  # ignore errors is the directory is not empty or if it does not exist
   rmdir "$TESTDIR" 2>/dev/null
fi

if [ -d "${TESTDIR}" ]; then
  echo "########################## Gathering results ##########################"
  export TESTDIR
  find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
    if [ ! -e "$i/LOG" ] ; then continue; fi

    cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
$(echo "###### $(cat "$i/INFO") ######")
$(echo "## test results in: \"$i\"")
$(grep -E -- "^(----|\*    diag)" "$i/LOG")
EOF
  done' sh {} +
fi

exit $_vtresult