devtool

#!/usr/bin/env bash

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# Firecracker devtool
#
# Use this script to build and test Firecracker.
#
# TL;DR
# Make sure you have Docker installed and properly configured
# (http://docker.com). Then,
#   building: `./devtool build`
#     Then find the binaries under build/debug/
#   testing: `./devtool test`
#     Will run the entire test battery; will take several minutes to complete.
#   deep-dive: `./devtool shell`
#     Open a shell prompt inside the container. Then build or test (or do
#     anything, really) manually.
#
# Still TL;DR: have Docker; ./devtool build; ./devtool test; ./devtool help.
#
#
# Both building and testing are done inside a Docker container. Please make sure
# you have Docker up and running on your system (see http:/docker.com) and your
# user has permission to run Docker containers.
#
# The Firecracker sources dir will be bind-mounted inside the development
# container (under /firecracker) and any files generated by the build process
# will show up under the build/ dir.  This includes the final binaries, as well
# as any intermediate or cache files.
#
# By default, all devtool commands run the container transparently, removing
# it after the command completes. Any persisting files will be stored under
# build/.
# If, for any reason, you want to access the container directly, please use
# `devtool shell`. This will perform the initial setup (bind-mounting the
# sources dir, setting privileges) and will then drop into a BASH shell inside
# the container.
#
# Building:
#   Run `./devtool build`.
#   By default, the debug binaries are built and placed under build/debug/.
#   To build the release version, run `./devtool build --release` instead.
#   You can then find the binaries under build/release/.
#
# Testing:
#   Run `./devtool test`.
#   This will run the entire integration test battery. The testing system is
#   based on pytest (http://pytest.org).
#
# Opening a shell prompt inside the development container:
#   Run `./devtool shell`.
#
# Additional information:
#   Run `./devtool help`.
#
#
# TODO:
#   - Cache test binaries, preserving them across `./devtool test` invocations.
#     At the moment, Firecracker is rebuilt everytime a test is run.
#   - List tests by parsing the `pytest --collect-only` output.
#   - Implement test filtering with `./devtool test --filter <filter>`
#   - Find an easier way to run individual tests on existing binaries.
#   - Add a `./devtool run` command to set up and run Firecracker.
#   - Add a `./devtool diag` command to help with troubleshooting, by checking
#     the most common failure conditions.
#   - Look into caching the Cargo registry within the container and if that
#     would help with reproducible builds (in addition to pinning Cargo.lock)

# Development container image (without tag)
DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm"

# Development container tag
DEVCTR_IMAGE_TAG="v51"

# Development container image (name:tag)
# This should be updated whenever we upgrade the development container.
# (Yet another step on our way to reproducible builds.)
DEVCTR_IMAGE="${DEVCTR_IMAGE_NO_TAG}:${DEVCTR_IMAGE_TAG}"

# Full path to the Firecracker tools dir on the host.
FC_TOOLS_DIR=$(cd "$(dirname "$0")" && pwd)
source "$FC_TOOLS_DIR/functions"

# Full path to the Firecracker sources dir on the host.
FC_ROOT_DIR=$(cd "${FC_TOOLS_DIR}/.." && pwd)

# Full path to the build dir on the host.
FC_BUILD_DIR="${FC_ROOT_DIR}/build"

# Full path to devctr dir on the host.
FC_DEVCTR_DIR="${FC_ROOT_DIR}/tools/devctr"

# Path to the linux kernel directory on the host.
KERNEL_DIR="${FC_ROOT_DIR}/.kernel"

# Full path to the cargo registry dir on the host. This appears on the host
# because we want to persist the cargo registry across container invocations.
# Otherwise, any rust crates from crates.io would be downloaded again each time
# we build or test.
CARGO_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_registry"

# Full path to the cargo git registry on the host. This serves the same purpose
# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
# crates.io.
CARGO_GIT_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_git_registry"

# Full path to the cargo target dir on the host.
CARGO_TARGET_DIR="${FC_BUILD_DIR}/cargo_target"

# Full path to the seccompiler cargo target dir on the host.
CARGO_SECCOMPILER_TARGET_DIR="${FC_BUILD_DIR}/seccompiler"

# Full path to the rebase-snap cargo target dir on the host.
CARGO_REBASE_SNAP_TARGET_DIR="${FC_BUILD_DIR}/rebase-snap"

# Full path to the Firecracker sources dir, as bind-mounted in the container.
CTR_FC_ROOT_DIR="/firecracker"

# Full path to the build dir, as bind-mounted in the container.
CTR_FC_BUILD_DIR="${CTR_FC_ROOT_DIR}/build"

# Full path to the cargo target dir, as bind-mounted in the container.
CTR_CARGO_TARGET_DIR="$CTR_FC_BUILD_DIR/cargo_target"

# Full path to the seccompiler cargo target dir, as bind-mounted in the container.
CTR_CARGO_SECCOMPILER_TARGET_DIR="$CTR_FC_BUILD_DIR/seccompiler"

# Full path to the rebase-snap cargo target dir, as bind-mounted in the container.
CTR_CARGO_REBASE_SNAP_TARGET_DIR="$CTR_FC_BUILD_DIR/rebase-snap"

# Full path to the microVM images cache dir
CTR_MICROVM_IMAGES_DIR="$CTR_FC_BUILD_DIR/img"

# Full path to the poetry tmp directory on the container, used for holding
# the lock and toml files.
CTR_POETRY_TMP_DIR="/tmp/poetry"

# Full path to the public key mapping on the guest
PUB_KEY_PATH=/root/.ssh/id_rsa.pub

# Full path to the private key mapping on the guest
PRIV_KEY_PATH=/root/.ssh/id_rsa

# Path to the linux kernel directory, as bind-mounted in the container.
CTR_KERNEL_DIR="${CTR_FC_ROOT_DIR}/.kernel"

# Global options received by $0
# These options are not command-specific, so we store them as global vars
OPT_UNATTENDED=false

# Get the target prefix to avoid repeated calls to uname -m
TARGET_PREFIX="$(uname -m)-unknown-linux-"

DEFAULT_TEST_SESSION_ROOT_PATH=/srv
DEFAULT_RAMDISK_PATH=/mnt/devtool-ramdisk


# Check if Docker is available and exit if it's not.
# Upon returning from this call, the caller can be certain Docker is available.
#
ensure_docker() {
    NEWLINE=$'\n'
    output=$(which docker 2>&1)
    ok_or_die "Docker not found. Aborting." \
        "Please make sure you have Docker (http://docker.com) installed" \
        "and properly configured.${NEWLINE}" \
        "Error: $?, command output: ${output}"

    output=$(docker ps 2>&1)
    ok_or_die "Error accessing Docker. Please make sure the Docker daemon" \
        "is running and that you are part of the docker group.${NEWLINE}" \
        "Error: $?, command output: ${output}${NEWLINE}" \
        "For more information, see" \
        "https://docs.docker.com/install/linux/linux-postinstall/"
}

# Run a command and retry multiple times if it fails. Once it stops
# failing return to normal execution. If there are "retry count"
# failures, set the last error code.
# $1 - command
# $2 - retry count
# $3 - sleep interval between retries
retry_cmd() {
    command=$1
    retry_cnt=$2
    sleep_int=$3

    {
        $command
    } || {
        # Command failed, substract one from retry_cnt
        retry_cnt=$((retry_cnt - 1))

        # If retry_cnt is larger than 0, sleep and call again
        if [ "$retry_cnt" -gt 0 ]; then
            echo "$command failed, retrying..."
            sleep "$sleep_int"
            retry_cmd "$command" "$retry_cnt" "$sleep_int"
        fi
    }
}

# Attempt to download our Docker image. Exit if that fails.
# Upon returning from this call, the caller can be certain our Docker image is
# available on this system.
#
ensure_devctr() {

    # We depend on having Docker present.
    ensure_docker

    # Check if we have the container image available locally. Attempt to
    # download it, if we don't.
    [[ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -gt 0 ]] || {
        say "About to pull docker image $DEVCTR_IMAGE"
        get_user_confirmation || die "Aborted."

        # Run docker pull 5 times in case it fails - sleep 3 seconds
        # between attempts
        retry_cmd "docker pull $DEVCTR_IMAGE" 5 3

        ok_or_die "Error pulling docker image. Aborting."
    }
}

# Make sure the build/ dirs are available. Exit if we can't create them.
# Upon returning from this call, the caller can be certain the build/ dirs exist.
#
ensure_build_dir() {
    for dir in "$FC_BUILD_DIR" "$CARGO_TARGET_DIR" \
               "$CARGO_REGISTRY_DIR" "$CARGO_GIT_REGISTRY_DIR"; do
        create_dir "$dir"
    done
}

build_fc_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_TARGET_DIR/$target/$profile/firecracker"
}

build_jailer_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_TARGET_DIR/$target/$profile/jailer"
}

build_seccomp_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin"
}

build_rebase_snap_bin_path() {
    target="$1"
    profile="$2"
    echo "$CARGO_REBASE_SNAP_TARGET_DIR/$target/$profile/rebase-snap"
}

ensure_release_binaries_exist() {
    target=$1
    profile=$2
    firecracker_bin_path=$( build_fc_bin_path "$target" "$profile")
    jailer_bin_path=$( build_jailer_bin_path "$target" "$profile")
    seccompiler_bin_path=$( build_seccomp_bin_path "$target" "$profile")
    rebase_snap_bin_path=$( build_rebase_snap_bin_path "$target" "$profile")

    { [ -f "$firecracker_bin_path" ] && [ -f "$jailer_bin_path" ] && [ -f "$seccompiler_bin_path" ] && \
    [ -f "$rebase_snap_bin_path" ]; } || \
    die "Missing release binaries. Needed files:\n" \
    "* $firecracker_bin_path\n" \
    "* $jailer_bin_path\n" \
    "* $seccompiler_bin_path\n" \
    "* $rebase_snap_bin_path\n" \
    "To build the binaries, run:\n\t$0 build --$profile"
}

# Fix build/ dir permissions after a privileged container run.
# Since the privileged container runs as root, any files it creates will be
# owned by root. This fixes that by recursively changing the ownership of build/
# to the current user.
#
cmd_fix_perms() {
    # Yes, running Docker to get elevated privileges, just to chown some files
    # is a dirty hack.
    run_devctr \
        -- \
        chown -R "$(id -u):$(id -g)" "$CTR_FC_BUILD_DIR"
}

# Builds the development container from its Dockerfile.
#
cmd_build_devctr() {
    arch=$(uname -m)
    docker_file_name="Dockerfile.$arch"
    build_args="--build-arg TMP_POETRY_DIR=$CTR_POETRY_TMP_DIR"

    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-n"|"--no-python-package-upgrade")
                shift
                build_args="$build_args --build-arg POETRY_LOCK_PATH=tools/devctr/poetry.lock"
                ;;
            "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    docker build -t "$DEVCTR_IMAGE_NO_TAG" -f "$FC_DEVCTR_DIR/$docker_file_name" $build_args .

    # Copy back the lockfile, since a new dependency or version would have
    # updated it.
    copy_poetry_lockfile
}


# Validate the user supplied kernel version number.
# It must be composed of 2 groups of integers separated by dot, with an optional third group.
validate_kernel_version() {
    local version_regex="^([0-9]+.)[0-9]+(.[0-9]+)?$"
    version="$1"

    if [ -z "$version" ]; then
        die "Kernel version cannot be empty."
    elif [[ ! "$version" =~ $version_regex ]]; then
        die "Invalid version number: $version (expected: \$Major.\$Minor.\$Patch(optional))."
    fi

}


# Helper function to run the dev container.
# Usage: run_devctr <docker args> -- <container args>
# Example: run_devctr --privileged -- bash -c "echo 'hello world'"
run_devctr() {
    docker_args=()
    ctr_args=()
    docker_args_done=false
    while [[ $# -gt 0 ]]; do
        [[ "$1" = "--" ]] && {
            docker_args_done=true
            shift
            continue
        }
        [[ $docker_args_done = true ]] && ctr_args+=("$1") || docker_args+=("$1")
        shift
    done

    # If we're running in a terminal, pass the terminal to Docker and run
    # the container interactively
    [[ -t 0 ]] && docker_args+=("-i")
    [[ -t 1 ]] && docker_args+=("-t")

    # Try to pass these environments from host into container for network proxies
    proxies=(http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY)
    for i in "${proxies[@]}"; do
        if [[ ! -z ${!i} ]]; then
            docker_args+=("--env") && docker_args+=("$i=${!i}")
        fi
    done

    # Finally, run the dev container
    # Use 'z' on the --volume parameter for docker to automatically relabel the
    # content and allow sharing between containers.
    docker run "${docker_args[@]}" \
        --rm \
        --volume /dev:/dev \
        --volume "$FC_ROOT_DIR:$CTR_FC_ROOT_DIR:z" \
        --env OPT_LOCAL_IMAGES_PATH="$(dirname "$CTR_MICROVM_IMAGES_DIR")" \
        --env PYTHONDONTWRITEBYTECODE=1 \
        "$DEVCTR_IMAGE" "${ctr_args[@]}"
}

# Helper function to test that the argument provided is a valid path to a SSH key.
#
test_key() {
    ssh-keygen -lf "$1" &>/dev/null
    ret=$?
    [ $ret -ne 0 ] && die "$1 is not a valid key file."
}

# Copies the ```poetry.lock``` file to the host, upgrading the versions if
# requested.
#
copy_poetry_lockfile() {
    # defined in Dockerfile
    lock_file_location_on_host="$FC_DEVCTR_DIR/poetry.lock"
    image_id=$(docker images -q "$DEVCTR_IMAGE_NO_TAG" | head -n 1)
    dummy_container_name=$(uuidgen)
    dummy_container_id=$(docker create --name "$dummy_container_name" "$image_id" bash)

    docker cp \
        "$dummy_container_id":"$CTR_POETRY_TMP_DIR"/poetry.lock \
        "$lock_file_location_on_host"

    docker rm -f "$dummy_container_name"
}

create_dir() {
    # Create a dir for the provided path.
    dir="$1"
    mkdir -p "$dir" || die "Error: cannot create dir $dir"
        [ -x "$dir" ] && [ -w "$dir" ] || \
            {
                say "Wrong permissions for $dir. Attempting to fix them ..."
                chmod +x+w "$dir"
            } || \
            die "Error: wrong permissions for $dir. Should be +x+w"
}

# `$0 help`
# Show the detailed devtool usage information.
#
cmd_help() {
    echo ""
    echo "Firecracker $(basename $0)"
    echo "Usage: $(basename $0) [<args>] <command> [<command args>]"
    echo ""
    echo "Global arguments"
    echo "    -y, --unattended         Run unattended. Assume the user would always"
    echo "                             answer \"yes\" to any confirmation prompt."
    echo ""
    echo "Available commands:"
    echo ""
    echo "    build [--debug|--release] [-l|--libc musl|gnu] [-- [<cargo args>]]"
    echo "        Build the Firecracker binaries."
    echo "        Firecracker is built using the Rust build system (cargo). All arguments after --"
    echo "        will be passed through to cargo."
    echo "        --debug               Build the debug binaries. This is the default."
    echo "        --release             Build the release binaries."
    echo "        -l, --libc musl|gnu   Choose the libc flavor against which Firecracker will"
    echo "                              be linked. Default is musl."
    echo "        --ssh-keys            Provide the paths to the public and private SSH keys on the host"
    echo "                              (in this particular order) required for the git authentication."
    echo "                              It is mandatory that both keys are specified."
    echo ""
    echo "    build_devctr [--no-python-package-upgrade]"
    echo "        Builds the development container from its Dockerfile."
    echo "        -n, --no-python-package-upgrade  Do not update python packages."
    echo ""
    echo "    build_kernel -c|--config [-n|--nproc]"
    echo "        Builds a kernel image custom-tailored for our CI."
    echo "        -c, --config  Path to the config file."
    echo "        -n, --nproc  Number of cores to use for building kernel."
    echo "    build_rootfs -s|--size [--partuuid]"
    echo "        Builds a rootfs image custom-tailored for use in our CI."
    echo "        -s, --size      Size of the rootfs image. Defaults to 300MB.
                                  The format is the same as that of 'truncates'."
    echo "        -p, --partuuid  Whether to build a partuuid image."
    echo ""
    echo "    checkenv"
    echo "        Performs prerequisites checks needed to execute firecracker."
    echo ""
    echo "    create_snapshot_artifacts"
    echo "        Runs a tool that generates snapshot artifacts for supported kernel versions."
    echo "        Snapshot mem and state files are saved under \`snapshot_artifacts/\` directory"
    echo ""
    echo "    distclean"
    echo "        Clean up the build tree and remove the docker container."
    echo ""
    echo "    fix_perms"
    echo "        Fixes permissions when devtool dies in the middle of a privileged session."
    echo ""
    echo "    fmt"
    echo "        Auto-format all Rust source files, to match the Firecracker requirements."
    echo "        This should be used as the last step in every commit, to ensure that the"
    echo "        Rust style tests pass."
    echo ""
    echo "    generate_syscall_tables <version>"
    echo "        Generates the syscall tables for seccompiler, according to a given kernel version."
    echo "        Release candidate (rc) linux versions are not allowed."
    echo "        Outputs a rust file for each supported arch: src/seccompiler/src/syscall_table/{arch}.rs"
    echo "        Supported architectures: x86_64 and aarch64."
    echo ""
    echo "    install [-p|--path] [--debug|--release]"
    echo "      Install firecracker, jailer and seccomp binaries to /usr/local/bin or a given path."
    echo "      Only the musl linked binaries are supported."
    echo "        --path                Install binaries to a specified path."
    echo "        --debug               Install the debug binaries."
    echo "        --release             Install the release binaries. This is the default."
    echo ""
    echo "    help"
    echo "        Display this help message."
    echo ""
    echo "    shell [--privileged]"
    echo "        Launch the development container and open an interactive BASH shell."
    echo "        -p, --privileged    Run the container as root, in privileged mode."
    echo "                            Running Firecracker via the jailer requires elevated"
    echo "                            privileges, though the build phase does not."
    echo ""
    echo "    test [-- [<pytest args>]]"
    echo "        Run the Firecracker integration tests."
    echo "        The Firecracker testing system is based on pytest. All arguments after --"
    echo "        will be passed through to pytest."
    echo "        -c, --cpuset-cpus cpulist    Set a dedicated cpulist to be used by the tests."
    echo "        -m, --cpuset-mems memlist    Set a dedicated memlist to be used by the tests."
    echo "        -r, --ramdisk size[k|m|g]    Use a ramdisk of `size` MB for
                                               the entire test session (e.g
                                               stored artifacts, Firecracker
                                               binaries, logs/metrics FIFOs
                                               and test created device files)."
    echo ""
}


# `$0 build` - build Firecracker
# Please see `$0 help` for more information.
#
cmd_build() {
    # By default, we'll build the debug binaries.
    profile="debug"
    libc="musl"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")  { cmd_help; exit 1;     } ;;
            "--debug")      { profile="debug";      } ;;
            "--release")    { profile="release";    } ;;
            "--ssh-keys")
                shift
                [[ -z "$1" ]] && \
                    die "Please provide the path to the public SSH key."
                [[ ! -f "$1" ]]  && die "The public key file does not exist: $1."
                test_key "$1"
                host_pub_key_path="$1"
                shift
                [[ -z "$1" ]] && \
                    die "Please provide the path to the private SSH key."
                [[ ! -f "$1" ]]  && die "The private key file does not exist: $1."
                test_key "$1"
                host_priv_key_path="$1"
                ;;
            "-l"|"--libc")
                shift
                [[ "$1" =~ ^(musl|gnu)$ ]] || \
                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
                libc="$1"
                ;;
            "--")           { shift; break;         } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check prerequisites
    ensure_devctr
    ensure_build_dir

    # Map the public and private keys to the guest if they are specified.
    [ ! -z "$host_pub_key_path" ] && [ ! -z "$host_priv_key_path" ] &&
        extra_args="--volume $host_pub_key_path:$PUB_KEY_PATH:z \
                    --volume $host_priv_key_path:$PRIV_KEY_PATH:z"

    # Run the cargo build process inside the container.
    # We don't need any special privileges for the build phase, so we run the
    # container as the current user/group.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${extra_args} \
        -- \
        ./tools/release.sh --libc $libc --profile $profile
    ret=$?
    return $ret
}

function cmd_make_release {
    cmd_test -- --json-report --json-report-file=test-report.json || die "Tests failed!"

    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- \
        ./tools/release.sh --libc musl --profile release --make-release
}

cmd_distclean() {
    # List of folders to remove.
    dirs=("build" "test_results")

    for dir in "${dirs[@]}"; do
        if [ -d "$dir" ]; then
            say "Removing $dir"
            rm -rf "$dir"
        fi
    done

    # Remove devctr if it exists
    if [ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -eq "1" ]; then
        say "Removing $DEVCTR_IMAGE"
        docker rmi -f "$DEVCTR_IMAGE"
    fi
}

mount_ramdisk() {
    local ramdisk_size="$1"
    umount_ramdisk
    DEFAULT_RAMDISK_PATH=$(sudo mktemp -d /mnt/devtool-ramdisk.XXXXXX)
    ok_or_die "Could not create ramdisk directory"
    sudo mkdir -p ${DEFAULT_RAMDISK_PATH} && \
    sudo mount -t tmpfs -o size=${ramdisk_size} tmpfs ${DEFAULT_RAMDISK_PATH}
    ok_or_die "Failed to mount ramdisk to ${DEFAULT_RAMDISK_PATH}. Check the permission."
    sudo mkdir -p ${DEFAULT_RAMDISK_PATH}/srv
    sudo mkdir -p  ${DEFAULT_RAMDISK_PATH}/tmp

    say "Using ramdisk: ${DEFAULT_RAMDISK_PATH}"
}

umount_ramdisk() {
    if [ ! -e "${DEFAULT_RAMDISK_PATH}" ]; then
        return 0
    fi
    if [ ! -d "${DEFAULT_RAMDISK_PATH}" ]; then
        die "${DEFAULT_RAMDISK_PATH} is not a directory."
    fi
    if [ ! -w "${DEFAULT_RAMDISK_PATH}" ]; then
        die "Failed to unmount ${DEFAULT_RAMDISK_PATH}. Check the permission."
    fi
    sudo umount ${DEFAULT_RAMDISK_PATH} &>/dev/null
    sudo rmdir ${DEFAULT_RAMDISK_PATH} &>/dev/null
}

# `$0 test` - run integration tests
# Please see `$0 help` for more information.
#
cmd_test() {

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-c"|"--cpuset-cpus")
                shift
                local cpuset_cpus="$1"
                ;;
            "-m"|"--cpuset-mems")
                shift
                local cpuset_mems="$1"
                ;;
            "-r"|"--ramdisk")
                shift
                local ramdisk_size="$1"
                local ramdisk=true
                ;;
            "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check prerequisites.
    ensure_kvm
    ensure_devctr
    ensure_build_dir

    # If we got to here, we've got all we need to continue.
    say "$(date -u +'%F %H:%M:%S %Z')"
    say "Kernel version: $(uname -r)"
    say "Starting test run ..."

    if [[ $ramdisk = true ]]; then
        mount_ramdisk ${ramdisk_size}
        ramdisk_args="--env TMPDIR=${DEFAULT_TEST_SESSION_ROOT_PATH}/tmp --volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
    fi

    # Testing (running Firecracker via the jailer) needs root access,
    # in order to set-up the Firecracker jail (manipulating cgroups, net
    # namespaces, etc).
    # We need to run a privileged container to get that kind of access.
    run_devctr \
        --privileged \
        --security-opt seccomp=unconfined \
        --ulimit core=0 \
        --ulimit nofile=4096:4096 \
        --ulimit memlock=-1:-1 \
        --workdir "$CTR_FC_ROOT_DIR/tests" \
        --cpuset-cpus="$cpuset_cpus" \
        --cpuset-mems="$cpuset_mems" \
        -v /boot:/boot \
        ${ramdisk_args} \
        -- \
        pytest "$@"

    ret=$?

    say "$(date -u +'%F %H:%M:%S %Z')"
    say "Finished test run ..."

    if [[ $ramdisk = true ]]; then
        umount_ramdisk
    fi

    # Running as root would have created some root-owned files under the build
    # dir. Let's fix that.
    cmd_fix_perms

    # Log CPU information
    say "$(lscpu)"

    return $ret
}

# `$0 shell` - drop to a shell prompt inside the dev container
# Please see `$0 help` for more information.
#
cmd_shell() {

    # By default, we run the container as the current user.
    privileged=false

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")          { cmd_help; exit 1; } ;;
            "-p"|"--privileged")    { privileged=true;  } ;;
            "-r"|"--ramdisk")
                  shift
                  local ramdisk_size="$1"
                  local ramdisk=true
                  ;;
              "--")               { shift; break;     } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Make sure we have what we need to continue.
    ensure_devctr
    ensure_build_dir

    if [[ $ramdisk = true ]]; then
        mount_ramdisk ${ramdisk_size}
        ramdisk_args="--volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
    fi

    if [[ $privileged = true ]]; then
        # If requested, spin up a privileged container.
        #
        say "Dropping to a privileged shell prompt ..."
        say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
        say_warn "You are running as root; any files that get created under" \
            "$CTR_FC_ROOT_DIR will be owned by root."
        run_devctr \
            --privileged \
            --ulimit nofile=4096:4096 \
            --ulimit memlock=-1:-1 \
            --security-opt seccomp=unconfined \
            --workdir "$CTR_FC_ROOT_DIR" \
            -v /boot:/boot \
            ${ramdisk_args} \
            -- \
            bash
        ret=$?

        # Running as root may have created some root-owned files under the build
        # dir. Let's fix that.
        #
        cmd_fix_perms
    else
        say "Dropping to shell prompt as user $(whoami) ..."
        say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
        say_warn "You won't be able to run Firecracker via the jailer," \
            "but you can still build it."
        say "You can use \`$0 shell --privileged\` to get a root shell."

        [ -w /dev/kvm ] || \
            say_warn "WARNING: user $(whoami) doesn't have permission to" \
                "access /dev/kvm. You won't be able to run Firecracker."

        run_devctr \
            --user "$(id -u):$(id -g)" \
            --ulimit nofile=4096:4096 \
            --ulimit memlock=-1:-1 \
            --device=/dev/kvm:/dev/kvm \
            --workdir "$CTR_FC_ROOT_DIR" \
            --env PS1="$(whoami)@\h:\w\$ " \
            -- \
            bash --norc
        ret=$?
    fi

    if [[ $ramdisk = true ]]; then
        umount_ramdisk
    fi

    return $ret
}


cmd_create_snapshot_artifacts() {
    privileged=true

    # Make sure we have what we need to continue.
    ensure_devctr
    ensure_build_dir

    say "Creating snapshot artifacts ..."
    run_devctr \
        --privileged \
        --ulimit nofile=4096:4096 \
        --ulimit memlock=-1:-1 \
        --security-opt seccomp=unconfined \
        --workdir "$CTR_FC_ROOT_DIR" \
        ${ramdisk_args} \
        -- \
        python3 tools/create_snapshot_artifact/main.py
    ret=$?

    # Running as root may have created some root-owned files under the build
    # dir. Let's fix that.
    cmd_fix_perms

    return $ret
}


# Auto-format all source code, to match the Firecracker requirements. For the
# moment, this is just a wrapper over `cargo fmt --all`
# Example: `devtool fmt`
#
cmd_fmt() {

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    ensure_devctr

    say "Applying rustfmt ..."
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- \
        cargo fmt --all
}


# Check if able to run firecracker.
# ../docs/getting-started.md#prerequisites
ensure_kvm_rw () {
    [[ -c /dev/kvm && -w /dev/kvm && -r /dev/kvm ]] || \
        say_err "FAILED: user $(whoami) doesn't have permission to" \
                "access /dev/kvm."
}

check_kernver () {
    KERN_MAJOR=4
    KERN_MINOR=14
    (uname -r | awk -v MAJOR=$KERN_MAJOR -v MINOR=$KERN_MINOR '{ split($0,kver,".");
    if( (kver[1] + (kver[2] / 100) ) <  MAJOR + (MINOR/100) )
    {
      exit 1;
    } }') ||
    say_err "FAILED: Kernel version must be >= $KERN_MAJOR.$KERN_MINOR"
}

# Check Production Host Setup
# ../docs/prod-host-setup.md

check_SMT () {
    (grep -q "^forceoff$\|^notsupported$" \
      /sys/devices/system/cpu/smt/control) ||
    say_warn "WARNING: Hyperthreading ENABLED."
}

check_KPTI () {
    (grep -q "^Mitigation: PTI$" \
      /sys/devices/system/cpu/vulnerabilities/meltdown) || \
    say_warn "WARNING: KPTI NOT SUPPORTED"
}

check_KSM () {
    (grep -q "^0$" /sys/kernel/mm/ksm/run) || \
    say_warn "WARNING: KSM ENABLED"
}

check_IBPB_IBRS () {
    (grep -q "^Mitigation: Full generic retpoline, IBPB, IBRS_FW$"\
      /sys/devices/system/cpu/vulnerabilities/spectre_v2) || \
    say_warn "WARNING: retpoline, IBPB, IBRS: DISABLED."
}

check_L1TF () {
    declare -a CONDITIONS=("Mitigation: PTE Inversion" "VMX: cache flushes")
    for cond in "${CONDITIONS[@]}";
    do (grep -q "$cond" /sys/devices/system/cpu/vulnerabilities/l1tf) ||
       say_warn "WARNING: $cond: DISABLED";
    done
}

check_swap () {
    (grep -q "swap.img" /proc/swaps ) && \
    say_warn "WARNING: SWAP ENABLED"
}

check_SSBD () {
    arch=$(uname -m)
    if [ "$arch" = "aarch64" ]; then
        local param="ssbd=force-on"
    elif [ "$arch" = "x86_64" ]; then
        local param="spec_store_bypass_disable=on"
    fi

    ssbd_sysfs_file="/sys/devices/system/cpu/vulnerabilities/spec_store_bypass"

    if [ -f "$ssbd_sysfs_file" ]; then
        (grep -q "^Vulnerable" $ssbd_sysfs_file) && \
        say_warn "WARNING: SSBD mitigation is either globally disabled or"\
            "system does not support mitigation via prctl or seccomp. Try"\
            "enabling it system-wide, using the \`${param}\` boot parameter."
    else
        say_warn "WARNING: SSBD mitigation not supported on this kernel."\
            "View the prod-host-setup.md for more details."
    fi
}

check_EPT() {
    if [ "$arch" = "x86_64" ]; then
        (grep -q "Y" /sys/module/kvm_intel/parameters/ept) || \
        say_warn "WARNING: EPT DISABLED. Performance will be affected."
    fi
}

check_vm() {
    if [ $(dmesg | grep -c -i "hypervisor detected") -gt 0 ]; then
        say_warn "WARNING: you are running in a virtual machine." \
    "Firecracker is not well tested under nested virtualization."
    fi
}

cmd_checkenv() {
    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
        ;;
        esac
        shift
    done
    PROD_DOC="../docs/prod-host-setup.md"
    QUICKSTART="../docs/getting-started.md#prerequisites"
    say "Checking prerequisites for running Firecracker."
    say "Please check $QUICKSTART in case of any error."
    ensure_kvm_rw
    check_kernver
    check_vm
    say "Checking Host Security Configuration."
    say "Please check $PROD_DOC in case of any error."
    check_KSM
    check_IBPB_IBRS
    check_L1TF
    check_SMT
    check_swap
    check_SSBD
    check_EPT
}

generate_syscall_table_x86_64() {
    path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/x86_64.rs"

    echo "$header" > $path_to_rust_file

    # the table for x86_64 is nicely formatted here: linux/arch/x86/entry/syscalls/syscall_64.tbl
    cat linux/arch/x86/entry/syscalls/syscall_64.tbl | grep -v "^#" | grep -v -e '^$' |\
        awk '{print $2,$3,$1}' | grep -v "^x32" |\
        awk '{print "    map.insert(\""$2"\".to_string(), "$3");"}' | sort >> $path_to_rust_file

    echo "$footer" >> $path_to_rust_file

    say "Generated at: $path_to_rust_file"
}

generate_syscall_table_aarch64() {
    path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/aarch64.rs"

    # filter for substituting `#define`s that point to other macros;
    # values taken from linux/include/uapi/asm-generic/unistd.h
    replace+='s/__NR3264_fadvise64/223/;'
    replace+='s/__NR3264_fcntl/25/;'
    replace+='s/__NR3264_fstatat/79/;'
    replace+='s/__NR3264_fstatfs/44/;'
    replace+='s/__NR3264_fstat/80/;'
    replace+='s/__NR3264_ftruncate/46/;'
    replace+='s/__NR3264_lseek/62/;'
    replace+='s/__NR3264_sendfile/71/;'
    replace+='s/__NR3264_statfs/43/;'
    replace+='s/__NR3264_truncate/45/;'
    replace+='s/__NR3264_mmap/222/;'

    echo "$header" > $path_to_rust_file

    # run the gcc command in the Docker container (to make sure that we have gcc installed)
    # the aarch64 syscall table is not located in a .tbl file, like x86; we run gcc's
    # pre-processor to extract the numeric constants from header files.
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$CTR_KERNEL_DIR" \
        -- \
            gcc -Ilinux/include/uapi -E -dM -D__ARCH_WANT_RENAMEAT\
                -D__BITS_PER_LONG=64\
                linux/arch/arm64/include/uapi/asm/unistd.h |\
                grep "#define __NR_" | grep -v "__NR_syscalls" |\
                grep -v "__NR_arch_specific_syscall" |\
                awk -F '__NR_' '{print $2}' |\
                sed $replace |\
                awk '{ print "    map.insert(\""$1"\".to_string(), "$2");" }' |\
                sort -d >> $path_to_rust_file
    ret=$?

    [ $ret -ne 0 ] && return $ret

    echo "$footer" >> $path_to_rust_file

    say "Generated at: $path_to_rust_file"
}

cmd_generate_syscall_tables() {
    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1;    } ;;
            *)                  { kernel_version="$1"; break; } ;;
        esac
        shift
    done

    validate_kernel_version "$kernel_version"

    kernel_major=v$(echo ${kernel_version} | cut -d . -f 1).x
    kernel_baseurl=https://www.kernel.org/pub/linux/kernel/${kernel_major}
    kernel_archive=linux-${kernel_version}.tar.xz

    ensure_devctr

    # Create the kernel clone directory
    rm -rf "$KERNEL_DIR"
    create_dir "$KERNEL_DIR"
    cd "$KERNEL_DIR"

    say "Fetching linux kernel..."

    # Get sha256 checksum.
    curl -fsSLO ${kernel_baseurl}/sha256sums.asc && \
    kernel_sha256=$(grep ${kernel_archive} sha256sums.asc | cut -d ' ' -f 1)
    # Get kernel archive.
    curl -fsSLO "$kernel_baseurl/$kernel_archive" && \
    # Verify checksum.
    echo "${kernel_sha256}  ${kernel_archive}" | sha256sum -c - && \
    # Decompress the kernel source.
    xz -d "${kernel_archive}" && \
    cat linux-${kernel_version}.tar | tar -x && mv linux-${kernel_version} linux

    ret=$?
    [ $ret -ne 0 ] && return $ret

    # rust file header
    read -r -d '' header << EOM
// Copyright $(date +"%Y") Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

// This file is auto-generated by \`tools/devtool generate_syscall_tables\`.
// Do NOT manually edit!
// Generated at: $(date)
// Kernel version: $kernel_version

use std::collections::HashMap;

pub(crate) fn make_syscall_table(map: &mut HashMap<String, i64>) {
EOM

    # rust file footer
    read -r -d '' footer << EOM
}

EOM

    # generate syscall table for x86_64
    say "Generating table for x86_64..."
    generate_syscall_table_x86_64 $header $footer

    # generate syscall table for aarch64
    say "Generating table for aarch64..."
    generate_syscall_table_aarch64 $header $footer

    ret=$?
    [ $ret -ne 0 ] && return $ret
}

cmd_install() {
    # By default we install release/musl binaries.
    profile="release"
    target="$TARGET_PREFIX""musl"
    install_path="/usr/local/bin"

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help") { cmd_help; exit 1; } ;;
            "-p"|"--path")
                shift;
                install_path=$1;
                ;;
            "--debug")      { profile="debug";      } ;;
            "--release")    { profile="release";    } ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # Check that the binaries exist first
    ensure_release_binaries_exist $target $profile

    say "Installing firecracker in $install_path"
    install -m 755 "$( build_fc_bin_path "$target" "$profile")" "$install_path"

    say "Installing jailer in $install_path"
    install -m 755 "$( build_jailer_bin_path "$target" "$profile")" "$install_path"

    say "Installing seccomp in $install_path"
    install -m 755 "$( build_seccomp_bin_path "$target" "$profile")" "$install_path"

    say "Installing rebase-snap in $install_path"
    install -m 755 "$( build_rebase_snap_bin_path "$target" "$profile")" "$install_path"
}

# Build a Firecracker CI compatible kernel image.
# Example: `./tools/devtool build_kernel -c resources/guest_configs/microvm-kernel-arm64-4.14.config`
#
cmd_build_kernel() {
    nprocs=$(getconf _NPROCESSORS_ONLN)

    # Parse any command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-c"|"--config")
                shift
                local KERNEL_CFG="$1"
                ;;
            "-n"|"--nproc")
                shift
                nprocs="$1"
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    # It is mandatory to provide a valid config file for building the kernel.
    [ -z "$KERNEL_CFG" ] && die "You need to provide the path to a valid kernel config!"

    arch=$(uname -m)
    if [ "$arch" = "x86_64" ]; then
        target="vmlinux"
        cfg_pattern="x86"
        format="elf"
    elif [ "$arch" = "aarch64" ]; then
        target="Image"
        cfg_pattern="arm64"
        format="pe"
    else
        die "Unsupported architecture!"
    fi

    kernel_dir_host="$FC_BUILD_DIR"/kernel
    kernel_dir_ctr="$CTR_FC_BUILD_DIR"/kernel
    create_dir "$kernel_dir_host"

    # Extract the kernel version from the config file provided as parameter.
    KERNEL_VERSION=$(grep -Po "^# Linux\/$cfg_pattern (([0-9]+.)[0-9]+)" "$KERNEL_CFG" | cut -d ' ' -f 3)
    validate_kernel_version "$KERNEL_VERSION"

    recipe_commit="b551cccc405a73a6d316c0c09dfe0b3e7a73ba3f"
    recipe_url="https://raw.githubusercontent.com/rust-vmm/vmm-reference/$recipe_commit/resources/kernel/make_kernel.sh"
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$kernel_dir_ctr" \
        -- /bin/bash -c "curl -LO $recipe_url && source make_kernel.sh && extract_kernel_srcs $KERNEL_VERSION"
    ok_or_die "Could not extract kernel sources with recipe $recipe_url!"

    cp "$KERNEL_CFG" "$kernel_dir_host/linux-$KERNEL_VERSION/.config"

    KERNEL_BINARY_NAME="vmlinux-$KERNEL_VERSION-$arch.bin"
    run_devctr \
        --user "$(id -u):$(id -g)" \
        --workdir "$kernel_dir_ctr" \
        -- /bin/bash -c "source make_kernel.sh && make_kernel "$kernel_dir_ctr/linux-"$KERNEL_VERSION"" $format $target $nprocs $KERNEL_BINARY_NAME"
    ok_or_die "Could not build kernel!"

    say "Successfully built kernel!"
    say "Kernel binary placed in: $kernel_dir_host/linux-$KERNEL_VERSION/$KERNEL_BINARY_NAME"
}

# `./devtool build_rootfs -s 500M`
# Build a rootfs of custom size.
#
cmd_build_rootfs() {
    # Default size for the resulting rootfs image is 300M.
    SIZE="300M"
    FROM_CTR=ubuntu:18.04
    flavour="bionic"
    ROOTFS_DIR=/firecracker/build/rootfs
    MNT_DIR=$ROOTFS_DIR/mnt

    # Parse the command line args.
    while [ $# -gt 0 ]; do
        case "$1" in
            "-h"|"--help")      { cmd_help; exit 1; } ;;
            "-s"|"--size")
                shift
                SIZE="$1"
                ;;
            "-p"|"--partuuid")
                shift
                PARTUUID=1
                ;;
            "-m"|"--mount")
                shift
                MNT_DIR="$1"
                ;;
            *)
                die "Unknown argument: $1. Please use --help for help."
            ;;
        esac
        shift
    done

    rootfs_dir_host="$FC_BUILD_DIR/rootfs"
    rootfs_dir_ctr="$CTR_FC_BUILD_DIR/rootfs"
    resources_dir_ctr="$CTR_FC_ROOT_DIR/resources/tests"
    ROOTFS_NAME="$flavour.rootfs.ext4"
    ROOTFS_PARTUUID_NAME="$flavour.rootfs_with_partuuid.ext4"

    create_dir "$rootfs_dir_host"

    run_devctr \
        --workdir "$CTR_FC_ROOT_DIR" \
        -- /bin/bash -c "gcc -o  $rootfs_dir_ctr/init $resources_dir_ctr/init.c && \
        gcc -o  $rootfs_dir_ctr/fillmem $resources_dir_ctr/fillmem.c && \
        gcc -o  $rootfs_dir_ctr/readmem $resources_dir_ctr/readmem.c"

    img_file="$rootfs_dir_host/$ROOTFS_NAME"

    # Check for pre-existence of rootfs image. See if we are allowed
    # to delete it. If not, abort.
    if [ -f "$img_file" ]; then
        say "Rootfs image $img_file already exists. Do you want us to delete it for you?"
        get_user_confirmation || die "Aborted."
        rm -f "$img_file"
    fi

    truncate -s "$SIZE" "$img_file"
    mkfs.ext4 -F "$img_file"
    docker run --env ROOTFS_DIR="$ROOTFS_DIR" --env MNT_DIR="$MNT_DIR" --env ROOTFS_NAME=$ROOTFS_NAME --env PARTUUID=$PARTUUID --privileged --rm -i -v "$FC_ROOT_DIR:/firecracker"  "$FROM_CTR" bash -s <<'EOF'
img_file="$ROOTFS_DIR/$ROOTFS_NAME"
resource_dir="/firecracker/resources/tests"
mnt_dir="$MNT_DIR"
dirs="bin etc home lib lib64 opt root sbin usr"

source $resource_dir/setup_rootfs.sh

mkdir -p $mnt_dir
echo "Mounting $img_file on $mnt_dir ..."
mount $img_file $mnt_dir

prepare_fc_rootfs "$ROOTFS_DIR" "$resource_dir"

# If we want to create a PARTUUID no need to install all packages (i.e cpuid, iperf3).
if [ -z "$PARTUUID" ]; then
    setup_specialized_rootfs "$ROOTFS_DIR" "$resource_dir"
    dirs="$dirs dev"
fi

# Copy everything we need to the bind-mounted rootfs image file
for d in $dirs; do tar c "/$d" | tar x -C $mnt_dir; done
umount $mnt_dir

EOF

    if [ -n "$PARTUUID" ]; then
        say "Now creating partuuid enabled rootfs..."

        run_devctr \
        --privileged \
        --workdir "$FC_DEVCTR_DIR" \
        -- /bin/bash -c "source "$CTR_FC_ROOT_DIR/resources/tests/setup_rootfs.sh" && create_partuuid_rootfs "$rootfs_dir_ctr/$ROOTFS_NAME" "$rootfs_dir_ctr/$ROOTFS_PARTUUID_NAME"" || die "Error while building partuuid enabled root!"
        label=$(blkid "$rootfs_dir_host/$ROOTFS_PARTUUID_NAME" | cut -d ' ' -f 2 | cut -d '=' -f 2)
        say "Created partuuid enabled disk with partition label: $label!"
        rm "$img_file"
        img_file="$rootfs_dir_host/$ROOTFS_PARTUUID_NAME"
    fi

    say "Successfully built rootfs!"
    say "Rootfs image binary and private key placed in $img_file and $rootfs_dir_host/ssh/id_rsa, respectively!"
}

main() {

    if [ $# = 0 ]; then
    die "No command provided. Please use \`$0 help\` for help."
    fi

    # Parse main command line args.
    #
    while [ $# -gt 0 ]; do
        case "$1" in
            -h|--help)              { cmd_help; exit 1;     } ;;
            -y|--unattended)        { OPT_UNATTENDED=true;  } ;;
            -*)
                die "Unknown arg: $1. Please use \`$0 help\` for help."
            ;;
            *)
                break
            ;;
        esac
        shift
    done

    # $1 is now a command name. Check if it is a valid command and, if so,
    # run it.
    #
    declare -f "cmd_$1" > /dev/null
    ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."

    cmd=cmd_$1
    shift

    # $@ is now a list of command-specific args
    #
    $cmd "$@"
}

main "$@"