feat: add replace_missing_with_na parameter (#9)

* Add `replace_missing_with_na` parameter to `homogenize_panel()` to allow panelcleaner to create
  variables in the waves that are missing their raw variables. This is helpful during data collection
  when not all variables have had submissions but you want to keep the original panel mapping
  specification.

Author: psanker

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: panelcleaner
 Title: An interactive interface to homogenize messy panel data into a long format
-Version: 0.0.4
+Version: 0.0.5
 Authors@R: 
     c(person(
             given = "Patrick",

NEWS.md

@@ -1,3 +1,10 @@
+# panelcleaner 0.0.5
+
+* Add `replace_missing_with_na` parameter to `homogenize_panel()` to allow panelcleaner to create
+  variables in the waves that are missing their raw variables. This is helpful during data collection
+  when not all variables have had submissions but you want to keep the original panel mapping
+  specification.
+  
 # panelcleaner 0.0.4
 
 * Fix bug where panelcleaner would not subset the variables to those that are present in the wave database if the user wants to allow missing variables to not stop homogenization

R/homogenization.R

@@ -42,6 +42,27 @@
 #' is really only useful if you intend you data to be used in a
 #' [blueprintr](https://nyuglobalties.github.io/blueprintr) project.
 #'
+#' # Extra Parameters
+#'
+#' In some cases the default behavior of panelcleaner is too restrictive, especially during
+#' the beginning of data collection. Often, APIs or general data exports don't include
+#' variables that don't have any submissions yet, but you still want to keep those variables
+#' in your input data. These parameters lift some restrictions on panelcleaner's behavior:
+#'
+#' * `drop_na_homogenized`: If `TRUE`, any NA entries in the homogenized_name column will be
+#'   ignored, as if the row in the panel mapping doesn't exist.
+#' * `ignored_missing_codings`: If `TRUE`, waves with NA codings but with non-NA homogenized
+#'   codings will not have their values homogenized.
+#' * `ignored_missing_homogenized_codings`: If `TRUE`, any variables that have defined wave
+#'   codings but no homogenized coding will not have their codings homogenized.
+#' * `error_missing_raw_variables`: If `FALSE`, raw variables that should be present in the
+#'   data, given the panel mapping, but aren't will not throw an error. Instead, they'll be
+#'   added to the list of [panelcleaner::issues()].
+#' * `replace_missing_with_na`: If `TRUE`, raw_variables that should be present in the data,
+#'   given the panel mapping, but are not will be created and filled with NA values. A message
+#'   will be displayed of all the variables where this action was applied. This value supersedes
+#'   `error_missing_raw_variables`.
+#'
 #' @export
 homogenize_panel <- function(panel, mapping = NULL, ...) {
   tk_assert(is_unhomogenized_panel(panel))
@@ -299,6 +320,7 @@ homogenize_wave_descriptions <- function(panel, w, long_map, ctx = list()) {
 
 homogenize_wave_names <- function(panel, w, long_map, ctx = list()) {
   error_missing_raw_variables <- ctx$error_missing_raw_variables %||% TRUE
+  replace_missing_with_na <- ctx$replace_missing_with_na %||% FALSE
 
   schema <- panel_mapping_schema(long_map)
 
@@ -315,12 +337,20 @@ homogenize_wave_names <- function(panel, w, long_map, ctx = list()) {
 
   if (any(!variables %in% names(wave_db))) {
     missing_vars <- long_map[!long_map[[schema$wave_name]] %in% names(wave_db), ][[schema$wave_name]]
+    missing_msg <- c(
+      "Some variables present in mapping for {ui_value(w)} are not in the data: [",
+      glue_collapse(ui_value(missing_vars), ", "), "]"
+    )
 
-    if (isTRUE(error_missing_raw_variables)) {
-      tk_err(c(
-        "Some variables present in mapping for {ui_value(w)} are not in the data: [",
-        glue_collapse(ui_value(missing_vars), ", "), "]"
-      ))
+    if (isTRUE(replace_missing_with_na)) {
+      msg <- c(missing_msg, "\nThey have been created with missingness in anticipation of their eventual existence")
+      message(paste0(msg, collapse = ""))
+
+      for (mv in missing_vars) {
+        wave_db[[mv]] <- NA
+      }
+    } else if (isTRUE(error_missing_raw_variables)) {
+      tk_err(missing_msg)
     } else {
       issue <- list(missing_vars)
       names(issue) <- glue("missing_raw_variables_{w}")

man/homogenize_panel.Rd

@@ -1,5 +1,3 @@
-context("Homogenization")
-
 test_that("Variable name homogenization works", {
   ids_1 <- sample(1:500, 100)
   ids_2 <- ids_1
@@ -374,3 +372,77 @@ test_that("Missing variables don't stop homogenization, if desired", {
     c("missing_raw_variables_t1", "missing_raw_variables_t2")
   )
 })
+
+test_that("Missing variables can be created, if desired", {
+  ids_1 <- sample(1:500, 100)
+  ids_2 <- ids_1
+  ids_2[sample(1:100, 10)] <- sample(500:1000, 10)
+
+  wave_1 <- data.frame(id = ids_1, time = 1, q1 = sample(1:5, 100, replace = TRUE), q2 = sample(0:1, 100, replace = TRUE), stringsAsFactors = FALSE)
+  wave_2 <- data.frame(id = ids_2, time = 2, question1 = sample(1:5, 100, replace = TRUE), Q2 = sample(0:1, 100, replace = TRUE), stringsAsFactors = FALSE)
+
+  coding_1 <- bquote(
+    coding(
+      code("Never", 1),
+      code("Rarely", 2),
+      code("Sometimes", 3),
+      code("Frequently", 4),
+      code("Always", 5)
+    )
+  )
+
+  coding_2 <- bquote(
+    coding(
+      code("Never", 5),
+      code("Rarely", 4),
+      code("Sometimes", 3),
+      code("Frequently", 2),
+      code("Always", 1)
+    )
+  )
+
+  coding_h <- bquote(
+    coding(
+      code("Never", 1),
+      code("Rarely", 2),
+      code("Sometimes", 3),
+      code("Frequently", 4),
+      code("Always", 5)
+    )
+  )
+
+  single_deparse <- function(expr) {
+    paste0(deparse(expr), collapse = "")
+  }
+
+  mapping <- tibble::tribble(
+    ~name_t1, ~coding_t1, ~name_t2, ~coding_t2, ~panel_name, ~homogenized_name, ~homogenized_coding,
+    "id", NA_character_, "id", NA_character_, "test_panel", "id", NA_character_,
+    "time", NA_character_, "time", NA_character_, "test_panel", "time", NA_character_,
+    "q1", NA_character_, "question1", NA_character_, "test_panel", "question_1", NA_character_,
+    "q2", NA_character_, "Q2", NA_character_, "test_panel", "question_2", NA_character_,
+    "q3", single_deparse(coding_1), "q3", single_deparse(coding_2), "test_panel", "question_3", single_deparse(coding_h)
+  )
+
+  panel_map <- panel_mapping(
+    mapping,
+    c("t1", "t2"),
+    .schema = list(
+      wave_name = "name",
+      wave_coding = "coding",
+      panel = "panel_name",
+      homogenized_name = "homogenized_name",
+      homogenized_coding = "homogenized_coding"
+    )
+  )
+
+  panel <-
+    enpanel("test_panel", t1 = wave_1, t2 = wave_2) %>%
+    add_mapping(panel_map)
+
+  # replace_missing_with_na overrides error_missing_raw_variables
+  homogenized_panel <- expect_no_error(homogenize_panel(panel, replace_missing_with_na = TRUE))
+
+  expect_true(all(is.na(wave(homogenized_panel, "t1")$question_3)))
+  expect_true(all(is.na(wave(homogenized_panel, "t2")$question_3)))
+})