diff --git a/R/api.R b/R/api.R index dcfdea8..067dea4 100644 --- a/R/api.R +++ b/R/api.R @@ -2,8 +2,6 @@ ##' ##' @title Create orderly runner ##' -##' @param root Orderly root -##' ##' @param repositories_base_path Path in which Git repositories are ##' cloned. ##' @@ -22,7 +20,7 @@ ##' ##' @export api <- function( - root, repositories_base_path, + repositories_base_path, validate = NULL, log_level = "info", skip_queue_creation = FALSE) { logger <- porcelain::porcelain_logger(log_level) @@ -31,12 +29,11 @@ api <- function( if (skip_queue_creation) { queue <- NULL } else { - queue <- Queue$new(root) + queue <- Queue$new() } api <- porcelain::porcelain$new(validate = validate, logger = logger) api$include_package_endpoints(state = list( - root = root, repositories_base_path = repositories_base_path, queue = queue)) api @@ -90,11 +87,13 @@ repository_branches <- function(repositories_base_path, url) { ##' @porcelain ##' GET /report/list => json(report_list) +##' query url :: string ##' query ref :: string -##' state root :: root -report_list <- function(root, ref) { - base <- git_remote_default_branch_ref(root) - reports <- get_reports(root = root, ref = ref, base = base) +##' state repositories_base_path :: repositories_base_path +report_list <- function(repositories_base_path, url, ref) { + repo <- repository_path(repositories_base_path, url) + base <- git_remote_default_branch_ref(repo) + reports <- get_reports(root = repo, ref = ref, base = base) data.frame( name = reports$name, @@ -105,11 +104,14 @@ report_list <- function(root, ref) { ##' @porcelain -##' GET /report//parameters => json(report_parameters) +##' GET /report/parameters => json(report_parameters) +##' query url :: string ##' query ref :: string -##' state root :: root -report_parameters <- function(root, ref, name) { - params <- get_report_parameters(name, ref, root) +##' query name :: string +##' state repositories_base_path :: repositories_base_path +report_parameters <- function(repositories_base_path, url, ref, name) { + repo <- repository_path(repositories_base_path, url) + params <- get_report_parameters(name, ref, repo) lapply(names(params), function(param_name) { value <- params[[param_name]] list( @@ -121,16 +123,17 @@ report_parameters <- function(root, ref, name) { ##' @porcelain ##' POST /report/run => json(report_run_response) -##' state root :: root ##' state queue :: queue ##' body data :: json(report_run_request) -submit_report_run <- function(root, queue, data) { +submit_report_run <- function(queue, data) { data <- jsonlite::parse_json(data) task_id <- queue$submit( data$name, + url = data$url, branch = data$branch, ref = data$hash, - parameters = data$parameters + parameters = data$parameters, + location = data$location ) list(taskId = scalar(task_id)) } diff --git a/R/git.R b/R/git.R index 154dc33..5ceb77e 100644 --- a/R/git.R +++ b/R/git.R @@ -110,3 +110,15 @@ git_diff_tree <- function(left, right, repo = NULL) { " (?[A-Z])(?\\d+)?\\t(?[^\\t]*)(?:\\t(?[^\\t]*))?$") as.data.frame(stringr::str_match(output, re)[, -1, drop = FALSE]) } + + +create_temporary_worktree <- function(repo, branch, tmpdir, env = parent.frame()) { + path <- withr::local_tempdir(tmpdir = tmpdir, .local_envir = env) + + git_run(c("worktree", "add", path, branch), repo = repo) + withr::defer(env = env, { + git_run(c("worktree", "remove", "--force", path), repo = repo) + }) + + path +} diff --git a/R/main.R b/R/main.R index f60cdfa..66f385a 100644 --- a/R/main.R +++ b/R/main.R @@ -1,6 +1,6 @@ parse_main <- function(args = commandArgs(TRUE)) { usage <- "Usage: -orderly.runner.server [options] +orderly.runner.server [options] Options: --log-level=LEVEL Log-level (off, info, all) [default: info] @@ -11,14 +11,13 @@ Options: list(log_level = dat$log_level, validate = dat$validate, port = as.integer(dat$port), - path = dat$path, - repositories = dat$repositories, + repositories = dat$path, host = dat$host) } main <- function(args = commandArgs(TRUE)) { dat <- parse_main(args) - api_obj <- api(dat$path, dat$repositories, dat$validate, dat$log_level) + api_obj <- api(dat$repositories, dat$validate, dat$log_level) api_obj$run(host = dat$host, port = dat$port) } @@ -33,16 +32,11 @@ orderly.runner.worker " main_worker <- function(args = commandArgs(TRUE)) { dat <- parse_main_worker(args) - # assumes ORDERLY_RUNNER_QUEUE_ID is set - queue <- Queue$new(dat$path) + queue_id <- Sys.getenv("ORDERLY_RUNNER_QUEUE_ID") + worker <- rrq::rrq_worker$new(queue_id, timeout_config = 30) - worker <- rrq::rrq_worker$new( - queue$controller$queue_id, - con = queue$controller$con - ) - worker_path <- file.path(dat$path, ".packit", "workers", worker$id) - fs::dir_create(worker_path) - gert::git_clone(dat$path, path = worker_path) + fs::dir_create(dat$path) - worker$loop() + # This environment variable is used by the code submitted by the API. + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = dat$path), worker$loop()) } diff --git a/R/porcelain.R b/R/porcelain.R index cbf63ae..478a435 100644 --- a/R/porcelain.R +++ b/R/porcelain.R @@ -34,18 +34,18 @@ "GET", "/report/list", report_list, - porcelain::porcelain_input_query(ref = "string"), - porcelain::porcelain_state(root = state$root), + porcelain::porcelain_input_query(url = "string", ref = "string"), + porcelain::porcelain_state(repositories_base_path = state$repositories_base_path), returning = porcelain::porcelain_returning_json("report_list"), validate = validate) }, - "GET /report//parameters" = function(state, validate) { + "GET /report/parameters" = function(state, validate) { porcelain::porcelain_endpoint$new( "GET", - "/report//parameters", + "/report/parameters", report_parameters, - porcelain::porcelain_input_query(ref = "string"), - porcelain::porcelain_state(root = state$root), + porcelain::porcelain_input_query(url = "string", ref = "string", name = "string"), + porcelain::porcelain_state(repositories_base_path = state$repositories_base_path), returning = porcelain::porcelain_returning_json("report_parameters"), validate = validate) }, @@ -55,7 +55,7 @@ "/report/run", submit_report_run, porcelain::porcelain_input_body_json("data", "report_run_request"), - porcelain::porcelain_state(root = state$root, queue = state$queue), + porcelain::porcelain_state(queue = state$queue), returning = porcelain::porcelain_returning_json("report_run_response"), validate = validate) }, diff --git a/R/queue.R b/R/queue.R index e13f8ba..f54ae09 100644 --- a/R/queue.R +++ b/R/queue.R @@ -4,29 +4,18 @@ Queue <- R6::R6Class("Queue", # nolint cloneable = FALSE, public = list( - #' @field root Orderly root - root = NULL, - #' @field config Orderly config - config = NULL, #' @field controller RRQ controller controller = NULL, #' @description #' Create object, read configuration and setup Redis connection. #' - #' @param root Orderly root. #' @param queue_id ID of an existing queue to connect to, creates a new one #' if NULL (default NULL) #' @param logs_dir directory to store worker logs - initialize = function(root, queue_id = NULL, logs_dir = "logs/worker") { - self$root <- root - self$config <- orderly2::orderly_config(self$root) - if (!runner_has_git(self$root)) { - cli::cli_abort(paste( - "Not starting server as orderly", - "root is not version controlled." - )) - } + initialize = function(queue_id = NULL, logs_dir = "logs/worker") { + # Connect to Redis + con <- redux::hiredis() # Create queue self$controller <- rrq::rrq_controller(queue_id %||% orderly_queue_id()) @@ -40,20 +29,21 @@ Queue <- R6::R6Class("Queue", # nolint #' @description #' Submit a job the Redis queue for runner to run. #' - #' @param reportname Name of orderly report. - #' @param parameters Parameters to run the report with (default NULL) - #' @param branch Name of git branch to checkout the repository - #' (default master) + #' @param url The URL of the Git repository containing the reports. + #' @param branch Name of git branch to checkout the repository. #' @param ref Git commit-ish value (e.g HEAD or commit hash or branch name). - #' We reset hard to this ref and run the report. (default HEAD) - submit = function(reportname, parameters = NULL, - branch = "master", ref = "HEAD") { + #' @param reportname Name of orderly report. + #' @param parameters Parameters to run the report with. + #' @param location Location of the outpack repository from which to pull + #' dependencies and push the produced packet. + submit = function(url, branch, ref, reportname, parameters, location) { run_args <- list( - self$root, + url, + branch, + ref, reportname, parameters, - branch, - ref + location ) rrq::rrq_task_create_call(runner_run, run_args, separate_process = TRUE, diff --git a/R/reports.R b/R/reports.R index c916886..a1d8550 100644 --- a/R/reports.R +++ b/R/reports.R @@ -41,8 +41,8 @@ get_report_parameters <- function(name, ref, root) { } -get_orderly_script_path <- function(name, ref, root) { - contents <- gert::git_ls(root, ref = ref) +get_orderly_script_path <- function(name, ref, repo) { + contents <- gert::git_ls(repo, ref = ref) re <- sprintf("^src/%s/(%s|orderly)\\.R$", name, name) matches <- grep(re, contents$path, value = TRUE, perl = TRUE) if (length(matches) != 1) { diff --git a/R/runner.R b/R/runner.R index fc66f0f..9c5317e 100644 --- a/R/runner.R +++ b/R/runner.R @@ -1,66 +1,34 @@ -runner_run <- function(orderly_root, reportname, parameters, branch, ref, ...) { - # Setup - worker_id <- Sys.getenv("RRQ_WORKER_ID") - worker_path <- file.path(orderly_root, ".packit", "workers", worker_id) - point_head_to_ref(worker_path, branch, ref) - - # Initial cleanup - git_clean(worker_path) - - # Run - id <- withr::with_envvar( - c(ORDERLY_SRC_ROOT = file.path(worker_path, "src", reportname)), - orderly2::orderly_run(reportname, parameters = parameters, - root = orderly_root, ...) - ) - - # Cleanup - git_clean(worker_path) - +runner_run <- function(url, branch, ref, reportname, parameters, location, ...) { + storage <- Sys.getenv("ORDERLY_WORKER_STORAGE") + stopifnot(nzchar(storage) && fs::dir_exists(storage)) + + repositories <- fs::dir_create(storage, "git") + worktree_base <- fs::dir_create(storage, "worktrees") + + repo <- git_sync(repositories, url) + + # We could create the worktree with a detached HEAD and not bother with + # creating the branch, but then Orderly's metadata wouldn't be as complete. + # + # Using a named branch does introduce some persistent state in the Git + # repository, which the runner generally does its best to avoid. That is an + # acceptable compromise given that repositories are private to each worker. + # + # The branch/ref association here does not have to match the remote: if + # the upstream repository has just been pushed to, the commit associated with + # this run may be an older commit of that branch. We will happily use that + # older commit. + gert::git_branch_create(branch, ref = ref, force = TRUE, checkout = FALSE, + repo = repo) + worktree <- create_temporary_worktree(repo, branch, worktree_base) + + orderly2::orderly_init(worktree) + orderly2::orderly_location_add("upstream", location$type, location$args, + root = worktree) + + id <- orderly2::orderly_run(reportname, parameters = parameters, + fetch_metadata = TRUE, allow_remote = TRUE, + location = "upstream", root = worktree, ...) + orderly2::orderly_location_push(id, "upstream", root = worktree) id } - -point_head_to_ref <- function(worker_path, branch, ref) { - gert::git_fetch(repo = worker_path) - gert::git_branch_checkout(branch, repo = worker_path) - gert::git_reset_hard(ref, repo = worker_path) -} - -add_dir_parent_if_empty <- function(files_to_delete, path) { - contained_files <- list.files(path, full.names = TRUE) - if (length(setdiff(contained_files, files_to_delete)) > 0) { - return(files_to_delete) - } - add_dir_parent_if_empty(c(files_to_delete, path), dirname(path)) -} - -get_empty_dirs <- function(worker_path) { - dirs <- fs::dir_ls(worker_path, recurse = TRUE, type = "directory") - Reduce(add_dir_parent_if_empty, c(list(character()), dirs)) -} - -git_clean <- function(worker_path) { - # gert does not have git clean but this should achieve the same thing - tryCatch( - { - gert::git_stash_save( - include_untracked = TRUE, - include_ignored = TRUE, - repo = worker_path - ) - gert::git_stash_drop(repo = worker_path) - }, - error = function(e) { - # we don't need to rethrow the error here since it doesn't break any - # further report runs - if (e$message != "cannot stash changes - there is nothing to stash.") { - # TODO add logger here - message(e$message) - } - NULL - } - ) - # however git ignores all directories, only cares about files, so we may - # have empty directories left - unlink(get_empty_dirs(worker_path), recursive = TRUE) -} diff --git a/docker/Dockerfile b/docker/Dockerfile index e39a6af..11d9337 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,10 +9,6 @@ COPY . /src RUN Rscript -e "pak::local_install('/src')" COPY docker/bin /usr/local/bin/ - -RUN git config --global --add safe.directory "*" -RUN echo ".packit" > /.gitignore -RUN git config --global core.excludesFile "/.gitignore" # ENTRYPOINT for server is /usr/local/bin/orderly.runner.server # ENTRYPOINT for worker is /usr/local/bin/orderly.runner.worker diff --git a/inst/schema/report_run_request.json b/inst/schema/report_run_request.json index 6f001d8..8de6679 100644 --- a/inst/schema/report_run_request.json +++ b/inst/schema/report_run_request.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "name": { + "url": { "type": "string" }, "branch": { @@ -11,6 +11,9 @@ "hash": { "type": "string" }, + "name": { + "type": "string" + }, "parameters": { "oneOf": [ { "type": "null" }, @@ -22,8 +25,19 @@ } } ] + }, + "location": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "args": { + "type": "object" + } + } } }, - "required": ["name", "branch", "hash", "parameters"], + "required": ["url", "branch", "hash", "name", "parameters", "location"], "additionalProperties": false } diff --git a/man/Queue.Rd b/man/Queue.Rd index 2d13465..f3d4e4a 100644 --- a/man/Queue.Rd +++ b/man/Queue.Rd @@ -12,10 +12,6 @@ Object for managing running jobs on the redis queue \section{Public fields}{ \if{html}{\out{
}} \describe{ -\item{\code{root}}{Orderly root} - -\item{\code{config}}{Orderly config} - \item{\code{controller}}{RRQ controller} } \if{html}{\out{
}} @@ -35,14 +31,12 @@ Object for managing running jobs on the redis queue \subsection{Method \code{new()}}{ Create object, read configuration and setup Redis connection. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{Queue$new(root, queue_id = NULL, logs_dir = "logs/worker")}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{Queue$new(queue_id = NULL, logs_dir = "logs/worker")}\if{html}{\out{
}} } \subsection{Arguments}{ \if{html}{\out{
}} \describe{ -\item{\code{root}}{Orderly root.} - \item{\code{queue_id}}{ID of an existing queue to connect to, creates a new one if NULL (default NULL)} @@ -57,21 +51,24 @@ if NULL (default NULL)} \subsection{Method \code{submit()}}{ Submit a job the Redis queue for runner to run. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{Queue$submit(reportname, parameters = NULL, branch = "master", ref = "HEAD")}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{Queue$submit(url, branch, ref, reportname, parameters, location)}\if{html}{\out{
}} } \subsection{Arguments}{ \if{html}{\out{
}} \describe{ -\item{\code{reportname}}{Name of orderly report.} +\item{\code{url}}{The URL of the Git repository containing the reports.} -\item{\code{parameters}}{Parameters to run the report with (default NULL)} +\item{\code{branch}}{Name of git branch to checkout the repository.} + +\item{\code{ref}}{Git commit-ish value (e.g HEAD or commit hash or branch name).} + +\item{\code{reportname}}{Name of orderly report.} -\item{\code{branch}}{Name of git branch to checkout the repository -(default master)} +\item{\code{parameters}}{Parameters to run the report with.} -\item{\code{ref}}{Git commit-ish value (e.g HEAD or commit hash or branch name). -We reset hard to this ref and run the report. (default HEAD)} +\item{\code{location}}{Location of the outpack repository from which to pull +dependencies and push the produced packet.} } \if{html}{\out{
}} } diff --git a/man/api.Rd b/man/api.Rd index 518c80b..f4d2dd4 100644 --- a/man/api.Rd +++ b/man/api.Rd @@ -5,7 +5,6 @@ \title{Create orderly runner} \usage{ api( - root, repositories_base_path, validate = NULL, log_level = "info", @@ -13,8 +12,6 @@ api( ) } \arguments{ -\item{root}{Orderly root} - \item{repositories_base_path}{Path in which Git repositories are cloned.} diff --git a/tests/testthat/examples/depends/depends.R b/tests/testthat/examples/depends/depends.R new file mode 100644 index 0000000..2e10fad --- /dev/null +++ b/tests/testthat/examples/depends/depends.R @@ -0,0 +1,3 @@ +orderly2::orderly_dependency("data", "latest", files = "data.rds") +numbers <- readRDS("data.rds") +saveRDS(sum(numbers), "sum.rds") diff --git a/tests/testthat/examples/git-clean/git-clean.R b/tests/testthat/examples/git-clean/git-clean.R deleted file mode 100644 index be10278..0000000 --- a/tests/testthat/examples/git-clean/git-clean.R +++ /dev/null @@ -1,5 +0,0 @@ -orderly2::orderly_artefact(description = "Some data", "data.rds") -d <- data.frame(a = 1:10, x = runif(10), y = 1:10 + runif(10)) -write.table("test", file = file.path("..", "..", "inside_draft.txt")) -write.table("test", file = file.path("..", "..", "..", "outside_draft.txt")) -saveRDS(d, "data.rds") diff --git a/tests/testthat/helper-orderly-runner.R b/tests/testthat/helper-orderly-runner.R index 0452a0b..36c27f2 100644 --- a/tests/testthat/helper-orderly-runner.R +++ b/tests/testthat/helper-orderly-runner.R @@ -1,5 +1,4 @@ -create_api <- function(root = NULL, - repositories = NULL, +create_api <- function(repositories = NULL, log_level = "off", ..., env = parent.frame()) { @@ -7,7 +6,7 @@ create_api <- function(root = NULL, repositories <- withr::local_tempdir(.local_envir = env) } - api(root, repositories, validate = TRUE, log_level = log_level, ...) + api(repositories, validate = TRUE, log_level = log_level, ...) } @@ -17,13 +16,6 @@ expect_success <- function(res) { } -create_temporary_root <- function(...) { - path <- tempfile() - withr::defer_parent(unlink(path, recursive = TRUE)) - suppressMessages(orderly2::orderly_init(path, ...)) -} - - skip_if_no_redis <- function() { available <- redux::redis_available() if (!available) { @@ -33,26 +25,30 @@ skip_if_no_redis <- function() { } -test_prepare_orderly_example <- function(examples, ..., env = parent.frame()) { +create_temporary_root <- function(..., env = parent.frame()) { path <- withr::local_tempdir(.local_envir = env) suppressMessages(orderly2::orderly_init(path, ...)) - copy_examples(examples, path) - - helper_add_git(path, orderly_gitignore = TRUE) - path } -test_prepare_orderly_remote_example <- function(examples, ..., env = parent.frame()) { - path_remote <- test_prepare_orderly_example(examples, ..., env = env) - path_local <- withr::local_tempdir(.local_envir = env) +test_prepare_orderly_example <- function(examples, ..., env = parent.frame()) { + # We explicitly do not initialize this as an orderly location, to mirror the + # fact that the source repository (eg. GitHub) is generally distinct from the + # upstream outpack repository (eg. Packit). + # + # We do still need to create orderly_config.yml as the bare minimum source + # tree. - gert::git_clone(path_remote, path_local) - orderly2::orderly_init(root = path_local, force = TRUE) - list( - remote = path_remote, - local = path_local - ) + path <- withr::local_tempdir(.local_envir = env) + writeLines('minimum_orderly_version: "1.99.0"', + file.path(path, "orderly_config.yml")) + + copy_examples(examples, path) + + gert::git_init(path) + git_add_and_commit(path, add = ".") + + path } @@ -66,67 +62,36 @@ copy_examples <- function(examples, path_src) { } -helper_add_git <- function(path, add = ".", orderly_gitignore = FALSE) { - gert::git_init(path) - if (orderly_gitignore) { - orderly2::orderly_gitignore_update("(root)", root = path) - } - sha <- git_add_and_commit(path, add) - branch <- gert::git_branch(repo = path) - url <- "https://example.com/git" - gert::git_remote_add(url, repo = path) - list(path = path, branch = branch, sha = sha, url = url) -} - -new_queue_quietly <- function(root, ...) { - suppressMessages(Queue$new(root, ...)) -} - -make_worker_dirs <- function(orderly_root, ids) { - packit_path <- file.path(orderly_root, ".packit") - dir.create(packit_path) - workers <- file.path(packit_path, "workers") - dir.create(workers) - lapply(ids, function(id) { - worker_path <- file.path(workers, id) - dir.create(worker_path) - gert::git_clone(orderly_root, path = worker_path) - gert::git_config_set("user.name", id, repo = worker_path) - gert::git_config_set("user.email", id, repo = worker_path) - }) +start_queue_workers <- function(n, controller, env = parent.frame()) { + storage <- withr::local_tempdir(.local_envir = env) + ids <- vcapply(seq(n), function(i) { + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = fs::dir_create(storage, i)), { + w <- suppressMessages(rrq::rrq_worker_spawn(1, controller = controller)) + w$id + }) + }) + withr::defer(rrq::rrq_worker_stop(ids, controller = controller), env = env) } -start_queue_workers_quietly <- function(n_workers, - controller, env = parent.frame()) { - worker_manager <- suppressMessages( - rrq::rrq_worker_spawn(n_workers, controller = controller) - ) - withr::defer(rrq::rrq_worker_stop(controller = controller), env = env) - worker_manager + +start_queue <- function(..., env = parent.frame()) { + logs_dir <- withr::local_tempdir(.local_envir = env) + Queue$new(logs_dir = logs_dir, ...) } -start_queue_with_workers <- function( - root, n_workers, env = parent.frame(), queue_id = NULL -) { - q <- new_queue_quietly(root, queue_id = queue_id, logs_dir = tempfile()) - worker_manager <- start_queue_workers_quietly(n_workers, q$controller, - env = env) - make_worker_dirs(root, worker_manager$id) + +start_queue_with_workers <- function(n, ..., env = parent.frame()) { + q <- start_queue(..., env = env) + start_queue_workers(n, q$controller, env = env) q } -skip_if_no_redis <- function() { - available <- redux::redis_available() - if (!available) { - testthat::skip("Skipping test as redis is not available") - } - invisible(available) -} expect_worker_task_complete <- function(task_id, controller, n_tries) { is_task_successful <- wait_for_task_complete(task_id, controller, n_tries) expect_true(is_task_successful) + invisible(get_task_result(task_id, controller)) } wait_for_task_complete <- function(task_id, controller, n_tries) { @@ -147,11 +112,12 @@ get_task_logs <- function(task_id, controller) { rrq::rrq_task_log(task_id, controller = controller) } -initialise_git_repo <- function() { - t <- tempfile() - dir.create(t) - writeLines(c("# Example", "", "example repo"), file.path(t, "README.md")) - helper_add_git(t) +initialise_git_repo <- function(env = parent.frame()) { + path <- gert::git_init(withr::local_tempdir(.local_envir = env)) + writeLines(c("# Example", "", "example repo"), + file.path(path, "README.md")) + git_add_and_commit(path, add = ".") + path } diff --git a/tests/testthat/test-api.R b/tests/testthat/test-api.R index 8f2fce1..a144d4f 100644 --- a/tests/testthat/test-api.R +++ b/tests/testthat/test-api.R @@ -1,6 +1,4 @@ test_that("Can construct the api", { - root <- create_temporary_root(use_file_store = TRUE) - obj <- create_api(log_level = "info", skip_queue_creation = TRUE) result <- evaluate_promise({ @@ -26,12 +24,12 @@ test_that("root data returns sensible data", { test_that("can fetch repositories", { - upstream_a <- test_prepare_orderly_example("data") - upstream_b <- test_prepare_orderly_example("data") - repositories <- withr::local_tempdir() obj <- create_api(repositories = repositories, skip_queue_creation = TRUE) + upstream_a <- test_prepare_orderly_example("data") + upstream_b <- test_prepare_orderly_example("data") + res <- obj$request( "POST", "/repository/fetch", @@ -51,10 +49,10 @@ test_that("can fetch repositories", { test_that("can list branches in repository", { - upstream <- test_prepare_orderly_example("data") - obj <- create_api(skip_queue_creation = TRUE) + upstream <- test_prepare_orderly_example("data") + # Start with just the initial master branch. Fetch the repo and list its # branches: res <- obj$request("POST", "/repository/fetch", @@ -105,9 +103,9 @@ test_that("can list branches in repository", { test_that("listing branches fails if repository was not fetched", { - upstream <- test_prepare_orderly_example("data") - obj <- create_api(skip_queue_creation = TRUE) + + upstream <- test_prepare_orderly_example("data") res <- obj$request("GET", "/repository/branches", query = list(url = upstream)) expect_equal(res$status, 404) @@ -115,10 +113,16 @@ test_that("listing branches fails if repository was not fetched", { test_that("can list orderly reports", { - repo <- test_prepare_orderly_remote_example(c("data", "parameters")) - obj <- create_api(root = repo$local, skip_queue_creation = TRUE) + obj <- create_api(skip_queue_creation = TRUE) - res <- obj$request("GET", "/report/list", query = list(ref = "master")) + upstream <- test_prepare_orderly_example(c("data", "parameters")) + + res <- obj$request("POST", "/repository/fetch", + body = jsonlite::toJSON(list(url = scalar(upstream)))) + expect_success(res) + + res <- obj$request("GET", "/report/list", query = list(url = upstream, + ref = "master")) data <- expect_success(res) expect_setequal(data$name, c("data", "parameters")) @@ -126,20 +130,26 @@ test_that("can list orderly reports", { expect_false(all(data$hasModifications)) ## Add a report on a 2nd branch - gert::git_branch_create("other", repo = repo$local) + gert::git_branch_create("other", repo = upstream) fs::dir_copy( - file.path(repo$local, "src", "parameters"), - file.path(repo$local, "src", "parameters2") + file.path(upstream, "src", "parameters"), + file.path(upstream, "src", "parameters2") ) # have to rename to parameters2.R to recognise it as orderly report file.rename( - file.path(repo$local, "src", "parameters2", "parameters.R"), - file.path(repo$local, "src", "parameters2", "parameters2.R") + file.path(upstream, "src", "parameters2", "parameters.R"), + file.path(upstream, "src", "parameters2", "parameters2.R") ) - sha <- git_add_and_commit(repo$local) + sha <- git_add_and_commit(upstream) + + ## Synchronize the local copy of the repository + res <- obj$request("POST", "/repository/fetch", + body = jsonlite::toJSON(list(url = scalar(upstream)))) + expect_success(res) ## Can list items from this sha - res <- obj$request("GET", "/report/list", query = list(ref = sha)) + res <- obj$request("GET", "/report/list", query = list(url = upstream, + ref = sha)) other_data <- expect_success(res) params2 <- other_data[other_data$name == "parameters2", ] existing <- res$data[other_data$name != "parameters2", ] @@ -148,21 +158,29 @@ test_that("can list orderly reports", { expect_true(params2$hasModifications) ## We can still see all reports on main branch - res <- obj$request("GET", "/report/list", query = list(ref = "master")) + res <- obj$request("GET", "/report/list", query = list(url = upstream, + ref = "master")) again_data <- expect_success(res) expect_equal(again_data, data) }) test_that("can get parameters for a report", { - repo <- test_prepare_orderly_remote_example(c("data", "parameters")) - obj <- create_api(root = repo$local, skip_queue_creation = TRUE) + obj <- create_api(skip_queue_creation = TRUE) + + upstream <- test_prepare_orderly_example(c("data", "parameters")) - res <- obj$request("GET", "/report/data/parameters", query = list(ref = "HEAD")) + res <- obj$request("POST", "/repository/fetch", + body = jsonlite::toJSON(list(url = scalar(upstream)))) + expect_success(res) + + res <- obj$request("GET", "/report/parameters", + query = list(url = upstream, ref = "HEAD", name = "data")) data <- expect_success(res) expect_equal(data, list()) - res <- obj$request("GET", "/report/parameters/parameters", query = list(ref = "HEAD")) + res <- obj$request("GET", "/report/parameters", + query = list(url = upstream, ref = "HEAD", name = "parameters")) data <- expect_success(res) expect_equal(data, data.frame(name = c("a", "b", "c"), value = c(NA, 2, NA))) @@ -175,20 +193,27 @@ test_that("can run orderly reports", { queue_id <- orderly_queue_id() controller <- rrq::rrq_controller(queue_id) - repo <- test_prepare_orderly_example(c("data", "parameters")) - obj <- withr::with_envvar( c(ORDERLY_RUNNER_QUEUE_ID = queue_id), - create_api(root = repo)) + create_api()) + + start_queue_workers(1, controller) - worker_manager <- start_queue_workers_quietly(1, controller) - make_worker_dirs(repo, worker_manager$id) + upstream_git <- test_prepare_orderly_example(c("data", "parameters")) + upstream_outpack <- create_temporary_root(use_file_store = TRUE) req <- list( + url = scalar(upstream_git), name = scalar("data"), - branch = scalar(gert::git_branch(repo = repo)), - hash = scalar(gert::git_commit_id(repo = repo)), - parameters = scalar(NULL) + branch = scalar(gert::git_branch(repo = upstream_git)), + hash = scalar(gert::git_commit_id(repo = upstream_git)), + parameters = scalar(NULL), + location = list( + type = scalar("path"), + args = list( + path = scalar(upstream_outpack) + ) + ) ) res <- obj$request("POST", "/report/run", body = jsonlite::toJSON(req)) @@ -200,10 +225,17 @@ test_that("can run orderly reports", { ) req <- list( + url = scalar(upstream_git), name = scalar("parameters"), - branch = scalar(gert::git_branch(repo = repo)), - hash = scalar(gert::git_commit_id(repo = repo)), - parameters = list(a = scalar(1), c = scalar(3)) + branch = scalar(gert::git_branch(repo = upstream_git)), + hash = scalar(gert::git_commit_id(repo = upstream_git)), + parameters = list(a = scalar(1), c = scalar(3)), + location = list( + type = scalar("path"), + args = list( + path = scalar(upstream_outpack) + ) + ) ) res <- obj$request("POST", "/report/run", body = jsonlite::toJSON(req)) @@ -221,20 +253,27 @@ test_that("can get statuses of jobs", { queue_id <- orderly_queue_id() controller <- rrq::rrq_controller(queue_id) - repo <- test_prepare_orderly_example(c("data", "parameters")) - obj <- withr::with_envvar( c(ORDERLY_RUNNER_QUEUE_ID = queue_id), - create_api(root = repo)) + create_api()) + + start_queue_workers(1, controller) - worker_manager <- start_queue_workers_quietly(1, controller) - make_worker_dirs(repo, worker_manager$id) + upstream_git <- test_prepare_orderly_example(c("data", "parameters")) + upstream_outpack <- create_temporary_root(use_file_store = TRUE) req <- list( + url = scalar(upstream_git), name = scalar("data"), - branch = scalar(gert::git_branch(repo = repo)), - hash = scalar(gert::git_commit_id(repo = repo)), - parameters = scalar(NULL) + branch = scalar(gert::git_branch(repo = upstream_git)), + hash = scalar(gert::git_commit_id(repo = upstream_git)), + parameters = scalar(NULL), + location = list( + type = scalar("path"), + args = list( + path = scalar(upstream_outpack) + ) + ) ) res1 <- obj$request("POST", "/report/run", body = jsonlite::toJSON(req)) diff --git a/tests/testthat/test-git.R b/tests/testthat/test-git.R index 7641701..7f32281 100644 --- a/tests/testthat/test-git.R +++ b/tests/testthat/test-git.R @@ -2,52 +2,45 @@ test_that("handle failure", { testthat::skip_on_cran() repo <- initialise_git_repo() expect_error( - git_run("unknown-command", repo = repo$path), + git_run("unknown-command", repo = repo), "'unknown-command' is not a git command", fixed = TRUE) }) -test_that("can get default branch when remote origin is set", { +test_that("can get default branch of clone", { testthat::skip_on_cran() - repo <- initialise_git_repo() - - expect_null(git_remote_default_branch_ref(repo$path)) - expect_null(git_remote_default_branch_name(repo$path)) - - git_run(c("symbolic-ref", - "refs/remotes/origin/HEAD", - "refs/remotes/origin/main"), - repo = repo$path) + upstream <- initialise_git_repo() + repo <- gert::git_clone(upstream, withr::local_tempdir()) - expect_equal(git_remote_default_branch_ref(repo$path), - "refs/remotes/origin/main") + expect_equal(git_remote_default_branch_ref(repo), + "refs/remotes/origin/master") - expect_equal(git_remote_default_branch_name(repo$path), - "main") + expect_equal(git_remote_default_branch_name(repo), + "master") }) test_that("can get last commit for a path", { repo <- initialise_git_repo() - c1 <- create_new_commit(repo$path, "hello.txt") - c2 <- create_new_commit(repo$path, "world.txt") - c3 <- create_new_commit(repo$path, "hello.txt") - c4 <- create_new_commit(repo$path, "world.txt") + c1 <- create_new_commit(repo, "hello.txt") + c2 <- create_new_commit(repo, "world.txt") + c3 <- create_new_commit(repo, "hello.txt") + c4 <- create_new_commit(repo, "world.txt") - expect_equal(git_get_latest_commit("hello.txt", "HEAD", repo$path), c3) - expect_equal(git_get_latest_commit("world.txt", "HEAD", repo$path), c4) + expect_equal(git_get_latest_commit("hello.txt", "HEAD", repo), c3) + expect_equal(git_get_latest_commit("world.txt", "HEAD", repo), c4) # If we start at c2, only it and its ancestors (ie. c1) are considered. - expect_equal(git_get_latest_commit("hello.txt", c2, repo$path), c1) - expect_equal(git_get_latest_commit("world.txt", c2, repo$path), c2) + expect_equal(git_get_latest_commit("hello.txt", c2, repo), c1) + expect_equal(git_get_latest_commit("world.txt", c2, repo), c2) }) test_that("can diff trees", { - repo <- test_prepare_orderly_remote_example("data") - copy_examples("parameters", repo$local) - git_add_and_commit(repo$local) + repo <- test_prepare_orderly_example("data") + copy_examples("parameters", repo) + git_add_and_commit(repo) - result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo$local) + result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo) expect_equal(nrow(result), 1) expect_equal(result$mode1, "000000") @@ -57,9 +50,9 @@ test_that("can diff trees", { expect_equal(result$status, "A") expect_equal(result$src, "parameters") - create_new_commit(repo$local, "src/parameters/hello.txt") + create_new_commit(repo, "src/parameters/hello.txt") - result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo$local) + result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo) expect_equal(nrow(result), 1) expect_equal(result$mode1, "040000") @@ -69,13 +62,13 @@ test_that("can diff trees", { expect_equal(result$status, "M") expect_equal(result$src, "parameters") - fs::file_move(file.path(repo$local, "src", "parameters"), - file.path(repo$local, "src", "zparameters")) - git_add_and_commit(repo$local) + fs::file_move(file.path(repo, "src", "parameters"), + file.path(repo, "src", "zparameters")) + git_add_and_commit(repo) # diff-tree never detects renames or copies. They are instead represented as # a add and delete. - result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo$local) + result <- git_diff_tree("HEAD^:src", "HEAD:src", repo = repo) expect_equal(nrow(result), 2) expect_equal(result$status[[1]], "D") expect_equal(result$src[[1]], "parameters") diff --git a/tests/testthat/test-main.R b/tests/testthat/test-main.R index 3fd66b2..37777f9 100644 --- a/tests/testthat/test-main.R +++ b/tests/testthat/test-main.R @@ -1,40 +1,35 @@ test_that("Can parse arguments (server)", { - expect_mapequal(parse_main(c("path", "repos")), + expect_mapequal(parse_main(c("path")), list(log_level = "info", validate = FALSE, port = 8001, host = "0.0.0.0", - path = "path", - repositories = "repos")) - expect_mapequal(parse_main(c("--port=8080", "path", "repos")), + repositories = "path")) + expect_mapequal(parse_main(c("--port=8080", "path")), list(log_level = "info", validate = FALSE, port = 8080, host = "0.0.0.0", - path = "path", - repositories = "repos")) - expect_mapequal(parse_main(c("--port=8080", "--validate", "path", "repos")), + repositories = "path")) + expect_mapequal(parse_main(c("--port=8080", "--validate", "path")), list(log_level = "info", validate = TRUE, port = 8080, host = "0.0.0.0", - path = "path", - repositories = "repos")) - expect_mapequal(parse_main(c("--log-level=debug", "--validate", "path", "repos")), + repositories = "path")) + expect_mapequal(parse_main(c("--log-level=debug", "--validate", "path")), list(log_level = "debug", validate = TRUE, port = 8001, host = "0.0.0.0", - path = "path", - repositories = "repos")) + repositories = "path")) expect_mapequal( - parse_main(c("--host=host", "--log-level=debug", "--validate", "path", "repos")), + parse_main(c("--host=host", "--log-level=debug", "--validate", "path")), list(log_level = "debug", validate = TRUE, port = 8001, host = "host", - path = "path", - repositories = "repos") + repositories = "path") ) }) @@ -44,11 +39,11 @@ test_that("Can construct api", { mock_run <- mockery::mock() mock_api <- mockery::mock(list(run = mock_run)) mockery::stub(main, "api", mock_api) - main(c("--host=my-host", "--log-level=debug", "path", "repositories")) + main(c("--host=my-host", "--log-level=debug", "path")) mockery::expect_called(mock_api, 1) expect_equal(mockery::mock_args(mock_api)[[1]], - list("path", "repositories", FALSE, "debug")) + list("path", FALSE, "debug")) mockery::expect_called(mock_run, 1) expect_equal(mockery::mock_args(mock_run)[[1]], @@ -56,53 +51,5 @@ test_that("Can construct api", { }) test_that("Can parse arguments (worker)", { - expect_mapequal(parse_main_worker("path"), - list(path = "path")) -}) - -test_that("Can spawn workers", { - skip_if_not_installed("mockery") - - mock_queue_new <- mockery::mock( - list(controller = list( - queue_id = "test_queue_id", - con = "test_con" - )) - ) - mockery::stub(main_worker, "Queue$new", mock_queue_new) - - mock_loop <- mockery::mock() - mock_rrq_worker_new <- mockery::mock( - list(loop = mock_loop, id = "test_worker_id") - ) - mockery::stub(main_worker, "rrq::rrq_worker$new", mock_rrq_worker_new) - - mock_dir_create <- mockery::mock() - mockery::stub(main_worker, "fs::dir_create", mock_dir_create) - - mock_git_clone <- mockery::mock() - mockery::stub(main_worker, "gert::git_clone", mock_git_clone) - - main_worker(c("path")) - - mockery::expect_called(mock_queue_new, 1) - expect_equal(mockery::mock_args(mock_queue_new)[[1]], - list("path")) - - mockery::expect_called(mock_rrq_worker_new, 1) - expect_equal(mockery::mock_args(mock_rrq_worker_new)[[1]], - list("test_queue_id", con = "test_con")) - - expected_worker_path <- file.path( - "path", ".packit", "workers", "test_worker_id" - ) - mockery::expect_called(mock_dir_create, 1) - expect_equal(mockery::mock_args(mock_dir_create)[[1]], - list(expected_worker_path)) - - mockery::expect_called(mock_git_clone, 1) - expect_equal(mockery::mock_args(mock_git_clone)[[1]], - list("path", path = expected_worker_path)) - - mockery::expect_called(mock_loop, 1) + expect_mapequal(parse_main_worker("path"), list(path = "path")) }) diff --git a/tests/testthat/test-queue.R b/tests/testthat/test-queue.R index c648c47..d0fad61 100644 --- a/tests/testthat/test-queue.R +++ b/tests/testthat/test-queue.R @@ -1,50 +1,34 @@ test_that("Can bring up queue", { skip_if_no_redis() - root <- create_temporary_root(use_file_store = TRUE) - gert::git_init(root) - q <- new_queue_quietly(root) - expect_equal(q$root, root) - expect_equal(q$config$core$use_file_store, TRUE) - expect_equal(q$controller, rrq::rrq_controller(q$controller$queue_id)) + q <- Queue$new() expect_equal(q$number_of_workers(), 0) - start_queue_workers_quietly(1, q$controller) + start_queue_workers(1, q$controller) expect_equal(q$number_of_workers(), 1) }) + test_that("creates directory for logs & adds to worker config", { skip_if_no_redis() - root <- create_temporary_root(use_file_store = TRUE) - gert::git_init(root) logs_dir <- tempfile() - q <- new_queue_quietly(root, logs_dir = logs_dir) - expect_true(dir.exists(logs_dir)) - expect_equal(logs_dir, rrq::rrq_worker_config_read("localhost", controller = q$controller)$logdir) -}) + q <- Queue$new(logs_dir = logs_dir) + expect_true(fs::dir_exists(logs_dir)) -test_that("Errors if not git repo", { - root <- create_temporary_root() - expect_error( - Queue$new(root), # nolint - paste( - "Not starting server as orderly root", - "is not version controlled." - ) - ) + config <- rrq::rrq_worker_config_read("localhost", controller = q$controller) + expect_equal(logs_dir, config$logdir) }) test_that("Can connect to existing queue with queue_id", { skip_if_no_redis() - root <- create_temporary_root() - gert::git_init(root) queue_id <- ids::random_id() - q1 <- new_queue_quietly(root, queue_id = queue_id) - start_queue_workers_quietly(1, q1$controller) - q2 <- new_queue_quietly(root, queue_id = queue_id) + q1 <- Queue$new(queue_id = queue_id) + start_queue_workers(1, q1$controller) + q2 <- Queue$new(queue_id = queue_id) + expect_equal(q1$number_of_workers(), 1) expect_equal(q2$number_of_workers(), 1) }) @@ -53,13 +37,10 @@ test_that("Can connect to existing queue with queue_id", { test_that("Uses ORDERLY_RUNNER_QUEUE_ID if it exists", { skip_if_no_redis() - root <- create_temporary_root() - gert::git_init(root) id <- ids::random_id() - q <- withr::with_envvar( - c(ORDERLY_RUNNER_QUEUE_ID = id), - new_queue_quietly(root) - ) + q <- withr::with_envvar(c(ORDERLY_RUNNER_QUEUE_ID = id), { + Queue$new() + }) expect_equal(q$controller$queue_id, id) }) @@ -67,9 +48,7 @@ test_that("Uses ORDERLY_RUNNER_QUEUE_ID if it exists", { test_that("Generated namespaced id if no ids exist", { skip_if_no_redis() - root <- create_temporary_root() - gert::git_init(root) - q <- new_queue_quietly(root) + q <- Queue$new() expect_match(q$controller$queue_id, "orderly.runner") }) @@ -77,68 +56,56 @@ test_that("Generated namespaced id if no ids exist", { test_that("Can submit task", { skip_if_no_redis() - root <- test_prepare_orderly_example("data") - - q <- start_queue_with_workers(root, 1) - - task_id <- q$submit("data", branch = gert::git_branch(root)) - expect_worker_task_complete(task_id, q$controller, 10) -}) - - -test_that("Can submit 2 tasks on different branches", { - skip_if_no_redis() - - root <- test_prepare_orderly_example("data") + upstream_git <- test_prepare_orderly_example("data") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) - gert::git_branch_create("branch", repo = root) - gert::git_branch_checkout("branch", repo = root) - create_new_commit(root, new_file = "test.txt", add = "test.txt") + q <- start_queue_with_workers(1) - q <- start_queue_with_workers(root, 2) - - task_id1 <- q$submit("data", branch = "master") - task_id2 <- q$submit("data", branch = "branch") - expect_worker_task_complete(task_id1, q$controller, 10) - expect_worker_task_complete(task_id2, q$controller, 10) + sha <- gert::git_commit_id(repo = upstream_git) + task_id <- q$submit( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)) + ) - worker_id2 <- rrq::rrq_task_info(task_id2, controller = q$controller)$worker - worker2_txt <- file.path(root, ".packit", "workers", worker_id2, "test.txt") - expect_equal(file.exists(worker2_txt), TRUE) + expect_worker_task_complete(task_id, q$controller, 10) }) -test_that("Can submit 2 tasks on different commit hashes", { +test_that("can get statuses on complete report runs with logs", { skip_if_no_redis() - root <- test_prepare_orderly_example("data") - commit1 <- gert::git_commit_id(repo = root) - commit2 <- create_new_commit(root, new_file = "test.txt", add = "test.txt") - - q <- start_queue_with_workers(root, 2) + upstream_git <- test_prepare_orderly_example("data") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) - task_id1 <- q$submit("data", ref = commit1, branch = gert::git_branch(root)) - task_id2 <- q$submit("data", ref = commit2, branch = gert::git_branch(root)) - expect_worker_task_complete(task_id1, q$controller, 10) - expect_worker_task_complete(task_id2, q$controller, 10) - - worker_id2 <- rrq::rrq_task_info(task_id2, controller = q$controller)$worker - worker2_txt <- file.path(root, ".packit", "workers", worker_id2, "test.txt") - expect_equal(file.exists(worker2_txt), TRUE) -}) + q <- start_queue_with_workers(1) + sha <- gert::git_commit_id(repo = upstream_git) + location <- list(type = "path", args = list(path = upstream_outpack)) + task_id1 <- q$submit( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = location + ) + task_id2 <- q$submit( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = location + ) -test_that("can get statuses on complete report runs with logs", { - skip_if_no_redis() - root <- test_prepare_orderly_example("data") - q <- start_queue_with_workers(root, 1) - task_id1 <- q$submit("data", branch = gert::git_branch(root)) - task_id2 <- q$submit("data", branch = gert::git_branch(root)) task_ids <- c(task_id1, task_id2) wait_for_task_complete(task_ids, q$controller, 5) statuses <- q$get_status(task_ids) - for (i in seq_along(task_ids)) { status <- statuses[[i]] expect_equal(status$status, scalar("COMPLETE")) @@ -147,20 +114,8 @@ test_that("can get statuses on complete report runs with logs", { expect_equal(status$logs, get_task_logs(task_ids[[i]], q$controller)) expect_equal(scalar(task_ids[[i]]), status$taskId) } -}) - -test_that("can get statuses wihtout logs if include_logs = false", { - # run 2 reports - skip_if_no_redis() - root <- test_prepare_orderly_example("data") - q <- start_queue_with_workers(root, 1) - task_id1 <- q$submit("data", branch = gert::git_branch(root)) - task_id2 <- q$submit("data", branch = gert::git_branch(root)) - task_ids <- c(task_id1, task_id2) - wait_for_task_complete(task_ids, q$controller, 5) - - statuses <- q$get_status(task_ids, FALSE) + statuses <- q$get_status(task_ids, include_logs = FALSE) for (i in seq_along(task_ids)) { status <- statuses[[i]] expect_equal(status$status, scalar("COMPLETE")) @@ -171,15 +126,35 @@ test_that("can get statuses wihtout logs if include_logs = false", { } }) + test_that("can get status on pending report run", { - # run 2 reports skip_if_no_redis() - root <- test_prepare_orderly_example("data") - q <- new_queue_quietly(root) - task_id1 <- q$submit("data", branch = gert::git_branch(root)) - task_id2 <- q$submit("data", branch = gert::git_branch(root)) - task_ids <- c(task_id1, task_id2) + upstream_git <- test_prepare_orderly_example("data") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) + + q <- start_queue() + + sha <- gert::git_commit_id(repo = upstream_git) + location <- list(type = "path", args = list(path = upstream_outpack)) + task_id1 <- q$submit( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = location + ) + task_id2 <- q$submit( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = location + ) + + task_ids <- c(task_id1, task_id2) statuses <- q$get_status(task_ids) for (i in seq_along(task_ids)) { diff --git a/tests/testthat/test-runner.R b/tests/testthat/test-runner.R index 5b30c52..10ffb49 100644 --- a/tests/testthat/test-runner.R +++ b/tests/testthat/test-runner.R @@ -1,64 +1,170 @@ test_that("runner runs as expected", { - orderly_root <- test_prepare_orderly_example("data") - - worker_id <- ids::adjective_animal() - make_worker_dirs(orderly_root, worker_id) - worker_root <- file.path(orderly_root, ".packit", "workers", worker_id) - - suppressMessages(withr::with_envvar( - c(RRQ_WORKER_ID = worker_id), - runner_run(orderly_root, "data", NULL, - gert::git_branch(orderly_root), - "HEAD", echo = FALSE) - )) - - # report has been run with data in archive - expect_equal(length(list.files(file.path(orderly_root, "archive"))), 1) - # cleanup has deleted draft folder - expect_equal(file.exists(file.path(worker_root, "draft")), FALSE) + upstream_git <- test_prepare_orderly_example("data") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) + + storage <- withr::local_tempdir() + + expect_false(fs::dir_exists(file.path(upstream_outpack, "archive"))) + + sha <- gert::git_commit_id(repo = upstream_git) + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage), suppressMessages({ + id <- runner_run( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + + # Runner has pushed the result to the upstream store + expect_true(fs::dir_exists( + file.path(upstream_outpack, "archive", "data", id))) + + info <- orderly2::orderly_metadata(id, root = upstream_outpack)$git + expect_equal(info$branch, "master") + expect_equal(info$sha, sha) }) + test_that("runner runs as expected with parameters", { - orderly_root <- test_prepare_orderly_example("parameters") + upstream_git <- test_prepare_orderly_example("parameters") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) - worker_id <- ids::adjective_animal() - make_worker_dirs(orderly_root, worker_id) - worker_root <- file.path(orderly_root, ".packit", "workers", worker_id) + storage <- withr::local_tempdir() + + expect_false(fs::dir_exists(file.path(upstream_outpack, "archive"))) parameters <- list(a = -1, b = -2, c = -3) - suppressMessages(withr::with_envvar( - c(RRQ_WORKER_ID = worker_id), - runner_run(orderly_root, "parameters", parameters, - gert::git_branch(orderly_root), - "HEAD", echo = FALSE) - )) + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage), suppressMessages({ + id <- runner_run( + url = upstream_git, + branch = "master", + ref = gert::git_commit_id(repo = upstream_git), + reportname = "parameters", + parameters = parameters, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) - report_archive <- file.path(orderly_root, "archive", "parameters") - rds_path <- file.path(report_archive, list.files(report_archive), "data.rds") - output <- readRDS(rds_path) + report <- file.path(upstream_outpack, "archive", "parameters", id) + output <- readRDS(file.path(report, "data.rds")) expect_equal(output, parameters) - expect_equal(file.exists(file.path(worker_root, "draft")), FALSE) }) -test_that("git clean clears unnecessary files", { - # git-clean.R spawns a file in draft folder and one in worker root folder - # and there will also be an empty folder draft/git-clean so we test - # all components of git_clean - orderly_root <- test_prepare_orderly_example("git-clean") - - worker_id <- ids::adjective_animal() - make_worker_dirs(orderly_root, worker_id) - worker_root <- file.path(orderly_root, ".packit", "workers", worker_id) - - suppressMessages(withr::with_envvar( - c(RRQ_WORKER_ID = worker_id), - runner_run(orderly_root, "git-clean", NULL, - gert::git_branch(orderly_root), - "HEAD", echo = FALSE) - )) - - expect_equal(length(list.files(file.path(orderly_root, "archive"))), 1) - expect_equal(file.exists(file.path(worker_root, "draft")), FALSE) - expect_equal(file.exists(file.path(worker_root, "outside_draft.txt")), FALSE) + +test_that("runner cleans up after itself", { + upstream_git <- test_prepare_orderly_example("data") + upstream_outpack <- create_temporary_root(use_file_store = TRUE) + + storage <- withr::local_tempdir() + + expect_false(fs::dir_exists(file.path(upstream_outpack, "archive"))) + + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage), suppressMessages({ + id <- runner_run( + url = upstream_git, + branch = "master", + ref = gert::git_commit_id(repo = upstream_git), + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + + # The only things left in the storage after running should be the local clone + # of the repository. The worktrees directory should be cleaned up and removed. + expect_setequal(fs::dir_ls(storage), + file.path(storage, c("git", "worktrees"))) + expect_length(fs::dir_ls(file.path(storage, "git")), 1) + expect_length(fs::dir_ls(file.path(storage, "worktrees")), 0) +}) + + +test_that("runner can use an old commit", { + upstream_git <- test_prepare_orderly_example("data") + commit1 <- gert::git_commit_id(repo = upstream_git) + commit2 <- create_new_commit(upstream_git) + + upstream_outpack <- create_temporary_root(use_file_store = TRUE) + + storage <- withr::local_tempdir() + + expect_false(fs::dir_exists(file.path(upstream_outpack, "archive"))) + + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage), suppressMessages({ + id1 <- runner_run( + url = upstream_git, + branch = "master", + ref = commit1, + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage), suppressMessages({ + id2 <- runner_run( + url = upstream_git, + branch = "master", + ref = commit2, + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + + info1 <- orderly2::orderly_metadata(id1, root = upstream_outpack)$git + expect_equal(info1$branch, "master") + expect_equal(info1$sha, commit1) + + info2 <- orderly2::orderly_metadata(id2, root = upstream_outpack)$git + expect_equal(info2$branch, "master") + expect_equal(info2$sha, commit2) +}) + + +test_that("runner can pull dependencies", { + upstream_git <- test_prepare_orderly_example(c("data", "depends")) + upstream_outpack <- create_temporary_root(use_file_store = TRUE) + + sha <- gert::git_commit_id(repo = upstream_git) + + # We use two separate storage directories to make sure we aren't just reusing + # the local packet somehow. + storage1 <- withr::local_tempdir() + storage2 <- withr::local_tempdir() + + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage1), suppressMessages({ + id1 <- runner_run( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "data", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + + withr::with_envvar(c(ORDERLY_WORKER_STORAGE = storage2), suppressMessages({ + id2 <- runner_run( + url = upstream_git, + branch = "master", + ref = sha, + reportname = "depends", + parameters = NULL, + location = list(type = "path", args = list(path = upstream_outpack)), + echo = FALSE) + })) + + # Runner has pushed the result to the upstream store + expect_true(fs::dir_exists( + file.path(upstream_outpack, "archive", "data", id1))) + expect_true(fs::dir_exists( + file.path(upstream_outpack, "archive", "depends", id2))) + + info <- orderly2::orderly_metadata(id2, root = upstream_outpack)$depends + expect_equal(info[1,]$packet, id1) + expect_equal(info[1,]$query, 'latest(name == "data")') }) diff --git a/tests/testthat/test-zzz-e2e.R b/tests/testthat/test-zzz-e2e.R index 6c899f8..9ae803a 100644 --- a/tests/testthat/test-zzz-e2e.R +++ b/tests/testthat/test-zzz-e2e.R @@ -2,19 +2,26 @@ skip_if_not_installed("httr") skip_if_no_redis() queue_id <- orderly_queue_id() -root <- test_prepare_orderly_remote_example( - c("data", "parameters") -) -repositories <- withr::local_tempdir() -queue <- start_queue_with_workers(root$local, 1, queue_id = queue_id) +queue <- start_queue_with_workers(1, queue_id = queue_id) + +upstream_git <- test_prepare_orderly_example(c("data", "parameters")) +upstream_outpack <- create_temporary_root(use_file_store = TRUE) + bg <- porcelain::porcelain_background$new( api, - args = list(root$local, repositories), + args = list(withr::local_tempdir()), env = c(ORDERLY_RUNNER_QUEUE_ID = queue_id) ) bg$start() on.exit(bg$stop()) +r <- bg$request("POST", + "/repository/fetch", + body = jsonlite::toJSON(list(url = scalar(upstream_git))), + encode = "raw", + httr::content_type("application/json")) +expect_equal(httr::status_code(r), 200) + test_that("can run server", { r <- bg$request("GET", "/") expect_equal(httr::status_code(r), 200) @@ -34,7 +41,7 @@ test_that("can run server", { test_that("can list reports", { - r <- bg$request("GET", "/report/list?ref=HEAD") + r <- bg$request("GET", sprintf("/report/list?url=%s&ref=HEAD", upstream_git)) expect_equal(httr::status_code(r), 200) dat <- httr::content(r) @@ -46,7 +53,7 @@ test_that("can list reports", { test_that("can get parameters", { - r <- bg$request("GET", "/report/data/parameters?ref=HEAD") + r <- bg$request("GET", sprintf("/report/parameters?url=%s&ref=HEAD&name=data", upstream_git)) expect_equal(httr::status_code(r), 200) dat <- httr::content(r) @@ -54,7 +61,7 @@ test_that("can get parameters", { expect_null(dat$errors) expect_equal(dat$data, list()) - r <- bg$request("GET", "/report/parameters/parameters?ref=HEAD") + r <- bg$request("GET", sprintf("/report/parameters?url=%s&ref=HEAD&name=parameters", upstream_git)) expect_equal(httr::status_code(r), 200) dat <- httr::content(r) @@ -70,9 +77,14 @@ test_that("can get parameters", { test_that("can run report", { data <- list( name = "data", - branch = gert::git_branch(repo = root$local), - hash = gert::git_commit_id(repo = root$local), - parameters = c(NULL) + url = upstream_git, + branch = gert::git_branch(repo = upstream_git), + hash = gert::git_commit_id(repo = upstream_git), + parameters = NULL, + location = list( + type = "path", + args = list(path = upstream_outpack) + ) ) body <- jsonlite::toJSON(data, null = "null", auto_unbox = TRUE) @@ -95,10 +107,15 @@ test_that("can run report", { test_that("can run report with params", { data <- list( + url = upstream_git, name = "parameters", - branch = gert::git_branch(repo = root$local), - hash = gert::git_commit_id(repo = root$local), - parameters = list(a = 1, c = 3) + branch = gert::git_branch(repo = upstream_git), + hash = gert::git_commit_id(repo = upstream_git), + parameters = list(a = 1, c = 3), + location = list( + type = "path", + args = list(path = upstream_outpack) + ) ) body <- jsonlite::toJSON(data, null = "null", auto_unbox = TRUE) @@ -137,9 +154,14 @@ test_that("can get status of report run with logs", { # run task and wait for finish before getting status data <- list( name = "data", - branch = gert::git_branch(repo = root$local), - hash = gert::git_commit_id(repo = root$local), - parameters = c(NULL) + url = upstream_git, + branch = gert::git_branch(repo = upstream_git), + hash = gert::git_commit_id(repo = upstream_git), + parameters = NULL, + location = list( + type = "path", + args = list(path = upstream_outpack) + ) ) r <- bg$request( "POST", "/report/run", @@ -174,9 +196,14 @@ test_that("can get status of multiple tasks without logs", { # run multiple tasks and wait for finish before getting status data <- list( name = "data", - branch = gert::git_branch(repo = root$local), - hash = gert::git_commit_id(repo = root$local), - parameters = c(NULL) + url = upstream_git, + branch = gert::git_branch(repo = upstream_git), + hash = gert::git_commit_id(repo = upstream_git), + parameters = NULL, + location = list( + type = "path", + args = list(path = upstream_outpack) + ) ) r1 <- bg$request( "POST", "/report/run", @@ -190,6 +217,9 @@ test_that("can get status of multiple tasks without logs", { encode = "raw", httr::content_type("application/json") ) + expect_equal(httr::status_code(r1), 200) + expect_equal(httr::status_code(r2), 200) + task_ids <- c(httr::content(r1)$data$taskId, httr::content(r2)$data$taskId) task_times <- wait_for_task_complete(task_ids, queue$controller, 3) @@ -219,10 +249,15 @@ test_that("can get status of multiple tasks without logs", { test_that("returns error with tasks ids of non-extant task ids", { # run report data <- list( + url = upstream_git, name = "data", - branch = gert::git_branch(repo = root$local), - hash = gert::git_commit_id(repo = root$local), - parameters = c(NULL) + branch = gert::git_branch(repo = upstream_git), + hash = gert::git_commit_id(repo = upstream_git), + parameters = NULL, + location = list( + type = "path", + args = list(path = upstream_outpack) + ) ) r1 <- bg$request( "POST", "/report/run", @@ -230,6 +265,7 @@ test_that("returns error with tasks ids of non-extant task ids", { encode = "raw", httr::content_type("application/json") ) + expect_equal(httr::status_code(r1), 200) task_ids <- c(httr::content(r1)$data$taskId, "non-existant-id") res <- bg$request(