diff --git a/.Rbuildignore b/.Rbuildignore index abace2b..6305600 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -34,3 +34,4 @@ ^vignettes/\.quarto$ ^vignettes/.*_files$ ^vignettes/.*\.html$ +^\.superpowers$ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 0000000..0858a8d --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,53 @@ +# R CMD check on every push to main/dev and every PR into them. +# Complements pkgdown.yaml, which only builds the site. Dependencies come +# from DESCRIPTION via setup-r-dependencies (Suggests included, so the +# optional simstudy backend is exercised here even when it is absent from +# a developer's local library). See .claude/AI.md for the CI lessons that +# shaped this file. +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + workflow_dispatch: + +name: R-CMD-check + +permissions: + contents: read + +jobs: + R-CMD-check: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + LANG: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck, any::roxygen2 + needs: check + + - name: Generate documentation + run: Rscript -e "roxygen2::roxygenize()" + + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + error-on: '"warning"' + build_args: 'c("--no-manual")' + args: 'c("--no-manual", "--as-cran")' diff --git a/.gitignore b/.gitignore index 51629cc..36eedeb 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ Generate_mock_data_files/ vignettes/.quarto/ vignettes/*_files/ vignettes/*.html + +# Superpowers plugin scratch (per-developer) +.superpowers/ diff --git a/NAMESPACE b/NAMESPACE index 987afd8..e6166e4 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,6 +12,7 @@ export(create_date_var) export(create_mock_data) export(create_survival_dates) export(create_wide_survival_data) +export(evaluate_mock_formulas) export(extract_distribution_params) export(extract_proportions) export(generate_garbage_values) @@ -31,10 +32,12 @@ export(make_garbage) export(mock_categorical) export(mock_continuous) export(mock_date) +export(mock_formula) export(mock_spec) export(mock_spec_categorical) export(mock_spec_continuous) export(mock_spec_date) +export(mock_spec_formula) export(mock_spec_from_recodeflow) export(parse_range_notation) export(parse_variable_start) diff --git a/NEWS.md b/NEWS.md index 634ba62..f80ef11 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,16 @@ # MockData (development version) +## Formula-derived variables (#39, Phase A) + +- Variables can now be generated from algebraic expressions over other + generated variables via a new `mockFormula` column in `variable_details` + (e.g. `weight / (height^2)`), or the new `mock_formula()` direct API. + Formulas are evaluated in dependency order by `evaluate_mock_formulas()`, + in a restricted environment exposing only the generated columns and a + fixed allow-list of base functions. `DerivedVar::`/`Func::` semantics are + unchanged: derived variables without a `mockFormula` remain excluded from + generation, and `Func::` dispatch is not yet supported. + ## Reproducibility (breaking change) - Seeded output changed once for the RNG-stream mechanism in this release. @@ -33,6 +44,23 @@ their generation. Native exponential values are truncated to the declared `range` by inverse-CDF sampling, rather than clipped at the range maximum (with a point mass at the boundary) as the legacy `rexp()` path does. +- The simstudy backend now returns a typed zero-row data frame for `n = 0`, + identical to the native backend's output, instead of failing inside + `simstudy::genData()` (whose `1:n` id table has two rows when `n = 0`). + (#50) +- The v0.4 pipeline now expands range-notation missing codes such as + `[997,999]` (the usual CCHS and CHMS pattern for don't know, refusal and + not stated) into their individual codes, splitting the row's proportion + equally. Previously a continuous variable with such a code failed in + post-processing, and a categorical variable silently wrote the literal + string `"[997,999]"` into the data. A bracketed missing code that is not + an integer range now fails with a message naming the variable. (#58) +- The packaged minimal example corrects two metadata errors: BMI's low + garbage range had a stray parenthesis (`[-10;15])`), and height's + low-garbage proportion was 1.00 (every row) instead of 0.01. With these + and #58, every variable in the example except its survival dates now + generates through the v0.4 pipeline. Seeded output for the example + changes. - With `validate = TRUE` (the default), invalid distribution parameters in metadata — e.g. `distribution = "exponential"` without a positive `rate` — now stop generation with a message naming the variable and how to fix it, diff --git a/R/create_mock_data.R b/R/create_mock_data.R index 3b28248..0817207 100644 --- a/R/create_mock_data.R +++ b/R/create_mock_data.R @@ -10,6 +10,15 @@ #' @noRd .create_mock_data_v04_unsupported_variables <- function(spec) { unsupported <- vapply(spec$variables, function(variable) { + # type = "formula" is supported end-to-end (#39, Phase A): baseline + # generation skips it, evaluate_mock_formulas() computes it, and + # postprocess_mock_data() applies its missing codes/garbage like any + # other variable. A stray non-empty `formula` field on a NON-formula + # variable is defensive fallback bait, not a supported feature. + if (variable$type == "formula") { + return(FALSE) + } + formula <- variable$formula has_formula <- !is.null(formula) && !(is.character(formula) && length(formula) == 1 && (is.na(formula) || trimws(formula) == "")) @@ -96,10 +105,12 @@ } baseline <- generate_mock_data_native(spec, n = n, seed = seed) - # Baseline and post-processing use distinct L'Ecuyer-CMRG sub-streams derived - # from the single public seed (see .with_mock_seed / ADR v05-seed-contract), - # so both stages pass the same seed and select their own stage internally. - postprocess_mock_data(baseline, spec, seed = seed) + staged <- evaluate_mock_formulas(baseline, spec, seed = seed) + # Baseline generation, formula evaluation, and post-processing use distinct + # L'Ecuyer-CMRG sub-streams derived from the single public seed (see + # .with_mock_seed / ADR v05-seed-contract), so all three stages pass the + # same seed and select their own stage internally. + postprocess_mock_data(staged, spec, seed = seed) } #' Create mock data from configuration files diff --git a/R/mock_spec.R b/R/mock_spec.R index 3b3df4c..04cb0fb 100644 --- a/R/mock_spec.R +++ b/R/mock_spec.R @@ -463,6 +463,73 @@ mock_date <- function(name, ) } +#' Create a direct formula-derived mock-data specification +#' +#' `mock_formula()` is the simple direct API for formula-derived variables. It +#' returns a validated `mock_spec`; it does not generate data. Formula +#' variables have no `range`/`levels`/`distribution` of their own — their +#' values come from evaluating `formula` over other variables in the same +#' specification once those have been generated; see +#' [evaluate_mock_formulas()]. +#' +#' @details +#' Use `mock_formula()` when specifying one formula variable directly in R +#' code. Use [mock_spec_formula()] with [mock_spec()] when composing several +#' variables or when writing an adapter from another metadata source. +#' +#' @param name Variable name. +#' @param formula Character scalar. An algebraic expression over other +#' variable names, e.g. `"weight / (height^2)"`. +#' @param rtype R output type for the computed column. Defaults to `"double"`. +#' @param missing_codes Explicit missing-code values. +#' @param missing_proportions Missing-code probabilities aligned to +#' `missing_codes`. +#' @param garbage_rules List of intentional invalid-value rules. +#' @param provenance Optional provenance metadata. Defaults to the direct API. +#' @param model_hint Backend hint. +#' @param spec_version Character version of the specification shape. +#' +#' @return A validated `mock_spec` object containing one formula variable. +#' @family direct specification APIs +#' @seealso [mock_spec()], [mock_spec_formula()], [evaluate_mock_formulas()] +#' +#' @examples +#' bmi_spec <- mock_spec( +#' mock_spec_continuous("height", range = c(1.4, 2.1)), +#' mock_spec_continuous("weight", range = c(45, 150)), +#' mock_spec_formula("bmi", formula = "weight / (height^2)") +#' ) +#' validate_mock_spec(bmi_spec) +#' +#' @export +mock_formula <- function(name, + formula, + rtype = "double", + missing_codes = numeric(0), + missing_proportions = numeric(0), + garbage_rules = list(), + provenance = NULL, + model_hint = "auto", + spec_version = .mock_spec_version) { + provenance <- .direct_api_provenance("mock_formula", provenance) + + mock_spec( + mock_spec_formula( + name = name, + formula = formula, + rtype = rtype, + missing_codes = missing_codes, + missing_proportions = missing_proportions, + garbage_rules = garbage_rules, + provenance = provenance, + model_hint = model_hint + ), + spec_version = spec_version, + provenance = provenance, + model_hint = model_hint + ) +} + #' Create a continuous variable specification #' #' @param name Variable name. @@ -627,6 +694,55 @@ mock_spec_date <- function(name, ) } +#' Create a formula-derived variable specification +#' +#' `mock_spec_formula()` describes a variable computed from other generated +#' variables by evaluating an algebraic expression (a `mockFormula`), rather +#' than sampled from a distribution. Evaluation happens in a restricted +#' environment exposing only the generated columns and a fixed allow-list of +#' base functions; see [evaluate_mock_formulas()]. +#' +#' @param name Variable name. +#' @param formula Character scalar. An algebraic expression over other +#' variable names, e.g. `"weight / (height^2)"`. +#' @param rtype R output type for the computed column. Defaults to `"double"`. +#' @param missing_codes,missing_proportions,garbage_rules,provenance,model_hint +#' As for other variable specifications; applied by post-processing. +#' +#' @return A `mock_spec_variable` object of type `"formula"`. +#' @family mock specification APIs +#' @seealso [mock_formula()], [evaluate_mock_formulas()] +#' @export +mock_spec_formula <- function(name, + formula, + rtype = "double", + missing_codes = numeric(0), + missing_proportions = numeric(0), + garbage_rules = list(), + provenance = "direct", + model_hint = "auto") { + variable <- .new_mock_spec_variable( + name = name, + type = "formula", + rtype = rtype, + formula = formula, + missing_codes = missing_codes, + missing_proportions = missing_proportions, + garbage_rules = garbage_rules, + provenance = provenance, + model_hint = model_hint + ) + # Parse failures are reported by validate_mock_spec(), not the constructor, + # matching the sibling constructors' defer-all-validation contract. + # depends_on is computed once here at construction time; it is not + # re-synced if `formula` is mutated afterwards. + variable$depends_on <- tryCatch( + .formula_dependencies(variable), + error = function(e) character(0) + ) + variable +} + #' Check whether an object is a MockData specification #' #' @param x Object to check. @@ -851,6 +967,27 @@ print.mock_spec_validation_result <- function(x, ...) { } } else if (variable$type == "date") { errors <- c(errors, .validate_range(variable$range, variable$name, "Date")) + } else if (variable$type == "formula") { + f <- variable$formula + if (is.null(f) || !is.character(f) || length(f) != 1 || is.na(f) || + trimws(f) == "") { + errors <- c(errors, paste0( + "Variable '", variable$name, + "' formula type requires a non-empty character formula." + )) + } else { + parse_error <- tryCatch({ str2lang(f); NULL }, error = function(e) conditionMessage(e)) + if (!is.null(parse_error)) { + errors <- c(errors, paste0( + "Variable '", variable$name, "' formula could not be parsed: ", parse_error + )) + } + } + if (!variable$rtype %in% c("double", "numeric", "integer", "factor", "character", "logical")) { + errors <- c(errors, paste0( + "Variable '", variable$name, "' has unsupported formula rType '", variable$rtype, "'." + )) + } } else { errors <- c(errors, paste0("Variable '", variable$name, "' has unsupported type '", variable$type, "'.")) } @@ -908,6 +1045,15 @@ validate_mock_spec <- function(spec, n = NULL, strict = TRUE) { for (variable in spec$variables) { errors <- c(errors, .validate_mock_spec_variable(variable)) } + + formula_error <- tryCatch({ + .validate_formula_referents(spec) + .order_formula_variables(spec) + NULL + }, error = function(e) conditionMessage(e)) + if (!is.null(formula_error)) { + errors <- c(errors, formula_error) + } } } diff --git a/R/mock_spec_formula.R b/R/mock_spec_formula.R new file mode 100644 index 0000000..bd0d22e --- /dev/null +++ b/R/mock_spec_formula.R @@ -0,0 +1,270 @@ +# ============================================================================== +# Formula-derived variables (#39, Phase A) — spec-side helpers. +# Evaluation itself lives in evaluate_mock_formulas() (same file, Task 2). +# ADR: development/adr/v05-formula-evaluator.md +# ============================================================================== + +# Base functions/operators a mockFormula may call. Deliberately closed and +# RNG-free: Phase A formulas are deterministic transformations of generated +# columns. Widen only on concrete need (ADR D3). +.formula_allowlist <- c( + "+", "-", "*", "/", "^", "%%", "%/%", "(", + "<", "<=", ">", ">=", "==", "!=", "&", "|", "!", + "ifelse", "pmin", "pmax", "min", "max", "abs", "round", "floor", + "ceiling", "sqrt", "exp", "log", "log10", "sum", "mean", "cut", + "as.numeric", "as.integer", "as.factor", "factor", "c" +) + +#' @noRd +.parse_mock_formula <- function(variable) { + tryCatch( + str2lang(variable$formula), + error = function(e) { + stop( + "Formula for variable '", variable$name, "' could not be parsed: ", + conditionMessage(e), + call. = FALSE + ) + } + ) +} + +#' @noRd +.is_formula_variable <- function(variable) { + identical(variable$type, "formula") +} + +#' @noRd +.formula_dependencies <- function(variable) { + if (!.is_formula_variable(variable)) { + return(character(0)) + } + all.vars(.parse_mock_formula(variable)) +} + +#' @noRd +.formula_function_symbols <- function(variable) { + expr <- .parse_mock_formula(variable) + setdiff(all.names(expr), all.vars(expr)) +} + +#' @noRd +.validate_formula_referents <- function(spec) { + spec_names <- names(spec$variables) + for (variable in spec$variables) { + if (!.is_formula_variable(variable)) next + # Unparseable formulas are reported once, by the per-variable validator + # branch (R/mock_spec.R) — re-parsing here (via .formula_dependencies()) + # would both duplicate that error and abort this loop before checking the + # OTHER, parseable formulas, masking their referent/symbol findings. + parseable <- tryCatch({ + .parse_mock_formula(variable) + TRUE + }, error = function(e) FALSE) + if (!parseable) next + missing <- setdiff(.formula_dependencies(variable), spec_names) + if (length(missing) > 0) { + stop( + "Formula for variable '", variable$name, + "' references unknown variable(s): ", + paste(missing, collapse = ", "), + call. = FALSE + ) + } + disallowed <- setdiff(.formula_function_symbols(variable), .formula_allowlist) + if (length(disallowed) > 0) { + stop( + "Formula for variable '", variable$name, + "' uses function(s) not permitted in mockFormula expressions: ", + paste(disallowed, collapse = ", "), + call. = FALSE + ) + } + } + invisible(TRUE) +} + +#' @noRd +.order_formula_variables <- function(spec) { + formula_names <- names(spec$variables)[ + vapply(spec$variables, .is_formula_variable, logical(1)) + ] + # Skip unparseable formulas here too (see .validate_formula_referents()): + # the per-variable validator already reports the parse error once, and a + # variable that can't be parsed has no computable dependency set — leaving + # it in `formula_names` would either duplicate the parse error (via + # .formula_dependencies()) or permanently block any variable that + # references it from ever being ordered. + parseable_names <- Filter(function(name) { + tryCatch({ + .parse_mock_formula(spec$variables[[name]]) + TRUE + }, error = function(e) FALSE) + }, formula_names) + + remaining <- parseable_names + ordered <- character(0) + while (length(remaining) > 0) { + progressed <- FALSE + for (name in remaining) { + deps <- intersect(.formula_dependencies(spec$variables[[name]]), parseable_names) + if (all(deps %in% ordered)) { + ordered <- c(ordered, name) + remaining <- setdiff(remaining, name) + progressed <- TRUE + } + } + if (!progressed) { + stop( + "Formula dependency cycle or unresolved ordering among: ", + paste(remaining, collapse = ", "), + call. = FALSE + ) + } + } + ordered +} + +#' @noRd +.formula_parent_env <- function() { + # baseNamespace() (the brief's original spelling) is not itself an + # exported/callable base function - .BaseNamespaceEnv is the base-R builtin + # binding for the same environment (verified: every .formula_allowlist + # symbol, including "(" and the operators, resolves via + # get(fn, envir = .BaseNamespaceEnv)). Its parent is emptyenv(), so nothing + # beyond this fixed set of base primitives is reachable from here. + env <- new.env(parent = emptyenv()) + for (fn in .formula_allowlist) { + env[[fn]] <- get(fn, envir = .BaseNamespaceEnv) + } + env +} + +#' Evaluate formula-derived variables over generated data +#' +#' Computes each `type = "formula"` variable in `spec` by evaluating its +#' expression over the columns of `data`, in dependency order. Evaluation runs +#' in a restricted environment exposing only the data columns and a fixed +#' allow-list of base functions — formulas cannot reach the caller's +#' environment. Any randomness would draw from the isolated `formula` +#' sub-stream (see the v0.5 seed contract), though the Phase A allow-list is +#' RNG-free, so evaluation is deterministic given `data`. +#' +#' @param data Data frame of generated baseline values (from +#' [generate_mock_data_native()] or [generate_mock_data_simstudy()]). +#' @param spec A `mock_spec`. Non-formula variables must already be columns of +#' `data`. Strict-validated at entry (like [postprocess_mock_data()]), so an +#' unparseable formula, an unknown referent, a disallowed symbol, or a +#' dependency cycle raises here rather than silently skipping the affected +#' variable. +#' @param seed Optional whole-number seed; reserved for future formulas with +#' random components. The caller's RNG state and kind are restored on exit. +#' +#' @return `data` with one appended column per formula variable. Returns +#' `data` unchanged if the spec has no formula variables. +#' @seealso [mock_formula()], [postprocess_mock_data()] +#' @export +evaluate_mock_formulas <- function(data, spec, seed = NULL) { + if (!is.data.frame(data)) { + stop("data must be a data frame.", call. = FALSE) + } + # Strict-validate first, exactly as postprocess_mock_data() does: the + # accumulate-and-skip behaviour inside .order_formula_variables() and + # .validate_formula_referents() is correct for validate_mock_spec()'s + # multi-error reporting, but it means an unparseable/cyclic/invalid-referent + # formula would otherwise be silently excluded from `ordered` below rather + # than reported — generation must never reach a bad formula. Strict + # validation already runs .validate_formula_referents()/ + # .order_formula_variables() (see validate_mock_spec()), so the direct call + # to .validate_formula_referents() that used to live below is redundant and + # has been removed. + validate_mock_spec(spec, n = nrow(data), strict = TRUE) + + ordered <- .order_formula_variables(spec) + if (length(ordered) == 0) { + return(data) + } + + non_formula <- setdiff(names(spec$variables), names(spec$variables)[ + vapply(spec$variables, .is_formula_variable, logical(1)) + ]) + missing_inputs <- setdiff(non_formula, names(data)) + if (length(missing_inputs) > 0) { + stop( + "data is missing generated column(s) required by formulas: ", + paste(missing_inputs, collapse = ", "), + call. = FALSE + ) + } + + .with_mock_seed(seed, stage = "formula", { + fn_env <- .formula_parent_env() + for (name in ordered) { + variable <- spec$variables[[name]] + values <- eval( + .parse_mock_formula(variable), + envir = as.list(data), + enclos = fn_env + ) + if (length(values) == 1 && nrow(data) != 1) { + values <- rep(values, nrow(data)) # scalar formulas broadcast + } + if (length(values) != nrow(data)) { + stop( + "Formula for variable '", name, "' returned length ", length(values), + ", expected ", nrow(data), ".", + call. = FALSE + ) + } + data[[name]] <- .coerce_formula_rtype(values, variable$rtype, name) + } + data + }) +} + +#' @noRd +.coerce_formula_rtype <- function(values, rtype, variable_name) { + switch(rtype, + double = { + if (is.factor(values)) { + warning( + "Variable '", variable_name, + "' formula produced a factor but rType is '", rtype, + "'; coercing to its integer level codes. Set rtype = \"factor\" (or \"character\") if that is not intended.", + call. = FALSE + ) + } + as.numeric(values) + }, + numeric = { + if (is.factor(values)) { + warning( + "Variable '", variable_name, + "' formula produced a factor but rType is '", rtype, + "'; coercing to its integer level codes. Set rtype = \"factor\" (or \"character\") if that is not intended.", + call. = FALSE + ) + } + as.numeric(values) + }, + integer = { + if (is.factor(values)) { + warning( + "Variable '", variable_name, + "' formula produced a factor but rType is '", rtype, + "'; coercing to its integer level codes. Set rtype = \"factor\" (or \"character\") if that is not intended.", + call. = FALSE + ) + } + as.integer(values) + }, + factor = as.factor(values), + character = as.character(values), + logical = as.logical(values), + stop( + "Variable '", variable_name, "' has unsupported formula rType '", + rtype, "'.", + call. = FALSE + ) + ) +} diff --git a/R/mock_spec_native.R b/R/mock_spec_native.R index e9425f5..7a29680 100644 --- a/R/mock_spec_native.R +++ b/R/mock_spec_native.R @@ -79,6 +79,14 @@ #' @noRd .native_formula_variables <- function(spec) { names(Filter(function(variable) { + # type == "formula" variables are supported (#39): the native backend + # skips them (see generate_mock_data_native()) and evaluate_mock_formulas() + # computes them post-baseline. This check only guards a *stray* formula + # field left on a variable of some OTHER type (e.g. an adapter mistake) - + # that combination remains an unsupported/fallback trigger. + if (.is_formula_variable(variable)) { + return(FALSE) + } formula <- variable$formula !is.null(formula) && !(is.character(formula) && length(formula) == 1 && (is.na(formula) || trimws(formula) == "")) @@ -333,8 +341,8 @@ #' #' `generate_mock_data_native()` consumes a validated `mock_spec` and generates #' baseline valid values using MockData's native R backend. This milestone does -#' not yet apply missing-code injection, garbage values, diagnostics, formula -#' evaluation, or optional `simstudy` features. +#' not yet apply missing-code injection, garbage values, diagnostics, or +#' optional `simstudy` features. #' #' @details #' The native backend is the default MIT-licensed baseline engine. It currently @@ -344,8 +352,14 @@ #' values, and diagnostics are intentionally handled by [postprocess_mock_data()] #' so that all backends share the same audit trail. #' -#' Formula variables are rejected loudly until the formula/dependency -#' milestone promotes the spike evaluator into production. +#' `type = "formula"` variables are skipped by this backend — they carry no +#' distribution to sample from. They are computed post-baseline by +#' [evaluate_mock_formulas()], which evaluates each formula over this +#' function's output columns in dependency order. A spec containing only +#' formula variables still returns an `n`-row, zero-column data frame here (see +#' [evaluate_mock_formulas()] for how columns are appended afterwards). A +#' stray `formula` field on a variable of some other type remains an +#' unsupported/fallback trigger. #' #' @param spec A `mock_spec` object. #' @param n Non-negative whole number of rows to generate. @@ -354,10 +368,13 @@ #' exit, so output is reproducible for a given seed and package version #' without perturbing the caller's RNG. #' -#' @return A data frame with one column per `mock_spec` variable and `n` rows. +#' @return A data frame with `n` rows and one column per non-formula +#' `mock_spec` variable (`type = "formula"` variables are appended +#' afterwards by [evaluate_mock_formulas()]). #' @family mock generation APIs #' @seealso [mock_spec()], [mock_continuous()], [mock_spec_from_recodeflow()], -#' [postprocess_mock_data()], [generate_mock_data_simstudy()] +#' [postprocess_mock_data()], [generate_mock_data_simstudy()], +#' [evaluate_mock_formulas()] #' #' @examples #' spec <- mock_spec( @@ -377,11 +394,21 @@ generate_mock_data_native <- function(spec, n, seed = NULL) { .check_native_backend_scope(spec) .with_mock_seed(seed, { - if (length(spec$variables) == 0) { + # type = "formula" variables have no distribution to sample from; they + # are computed post-baseline by evaluate_mock_formulas(). Filtering them + # out here (rather than in .generate_native_variable()) leaves this + # branch's assembly of `columns` byte-identical to before #39 for any + # spec with no formula variables (the common, zero-seeded-output-change + # case) - and a formula-only spec still falls through to the same + # .empty_native_data(n) path a variable-less spec would use. + generated_variables <- spec$variables[ + !vapply(spec$variables, .is_formula_variable, logical(1)) + ] + if (length(generated_variables) == 0) { .empty_native_data(n) } else { - columns <- lapply(spec$variables, .generate_native_variable, n = n) - names(columns) <- names(spec$variables) + columns <- lapply(generated_variables, .generate_native_variable, n = n) + names(columns) <- names(generated_variables) as.data.frame(columns, stringsAsFactors = FALSE, check.names = FALSE) } }, stage = "baseline") diff --git a/R/mock_spec_postprocess.R b/R/mock_spec_postprocess.R index da6c9f9..cb19225 100644 --- a/R/mock_spec_postprocess.R +++ b/R/mock_spec_postprocess.R @@ -23,7 +23,7 @@ garbage_rule_names ) - list( + entry <- list( n = n, preexisting_missing_code_indices = integer(0), assigned_missing_indices = integer(0), @@ -31,6 +31,17 @@ assigned_garbage_indices = garbage_indices, assigned_garbage_values = garbage_values ) + + # D5 (#39, Phase A): formula variables get diagnostics fields marking + # them as derived, so downstream consumers can tell a computed column + # from a sampled one without re-deriving it from the spec. + if (.is_formula_variable(variable)) { + entry$derived <- TRUE + entry$formula <- variable$formula + entry$depends_on <- variable$depends_on + } + + entry }) list( diff --git a/R/mock_spec_recodeflow.R b/R/mock_spec_recodeflow.R index aa85c70..0ba13ae 100644 --- a/R/mock_spec_recodeflow.R +++ b/R/mock_spec_recodeflow.R @@ -221,7 +221,45 @@ } #' @noRd -.recodeflow_missing <- function(details) { +.expand_range_missing_codes <- function(codes, proportions, variable) { + # #58: recodeflow writes grouped missing codes in range notation, e.g. + # "[997,999]" for don't know / refusal / not stated. Expand an integer range + # into its individual codes, splitting the row's proportion equally (the + # legacy generators draw uniformly within the range). Any other bracketed + # code cannot be inserted as a value, so fail naming the variable rather + # than writing the literal string into the data. + codes <- as.character(codes) + expanded_codes <- character(0) + expanded_proportions <- numeric(0) + for (i in seq_along(codes)) { + code <- trimws(codes[i]) + if (!startsWith(code, "[") && !startsWith(code, "(")) { + expanded_codes <- c(expanded_codes, code) + expanded_proportions <- c(expanded_proportions, proportions[i]) + next + } + parsed <- parse_range_notation(code) + if (is.null(parsed) || !identical(parsed$type, "integer") || + length(parsed$values) == 0) { + stop( + "Variable '", variable, "' has a range missing code '", code, + "' that is not an integer range. Write it as an integer range such ", + "as [997,999], or list each code on its own row.", + call. = FALSE + ) + } + values <- as.character(parsed$values) + expanded_codes <- c(expanded_codes, values) + expanded_proportions <- c( + expanded_proportions, + rep(proportions[i] / length(values), length(values)) + ) + } + list(codes = expanded_codes, proportions = expanded_proportions) +} + +#' @noRd +.recodeflow_missing <- function(details, variable = "") { if (is.null(details) || nrow(details) == 0 || !"recEnd" %in% names(details)) { return(list(codes = character(0), proportions = numeric(0))) } @@ -239,12 +277,43 @@ proportions <- if ("proportion" %in% names(missing_rows)) missing_rows$proportion else rep(NA_real_, nrow(missing_rows)) proportions[is.na(proportions)] <- 0 - list( - codes = as.character(missing_rows$recStart), - proportions = as.numeric(proportions) + .expand_range_missing_codes( + missing_rows$recStart, + as.numeric(proportions), + variable ) } +#' @noRd +# Formula-derived variables (#39, Phase A). A variable's mockFormula is a +# variable-level fact but is stored row-level on variable_details (ADR +# v05-formula-evaluator D1(a)): a variable's details may span multiple rows +# (e.g. one DerivedVar:: row plus missing-code rows), so read the first +# non-blank mockFormula among them. Returns "" when the column is absent or +# every value is blank/NA. +.details_mock_formula <- function(details) { + if (is.null(details) || nrow(details) == 0 || !"mockFormula" %in% names(details)) { + return("") + } + + values <- as.character(details$mockFormula) + values <- values[!vapply(values, .is_blank, logical(1))] + if (length(values) == 0) { + return("") + } + + # Warn if there are conflicting (distinct) mockFormula values + if (length(unique(values)) > 1) { + warning( + "Variable has conflicting mockFormula values across detail rows; using the first: '", + trimws(values[1]), "'.", + call. = FALSE + ) + } + + trimws(values[1]) +} + #' @noRd .recodeflow_distribution <- function(var_row, details) { distribution <- tolower(.row_character(var_row, "distribution", "")) @@ -308,22 +377,47 @@ kind <- .recodeflow_variable_kind(var_row) rtype <- .recodeflow_rtype(var_row, kind) provenance <- .recodeflow_provenance(variable, databaseStart) - missing <- .recodeflow_missing(details) + missing <- .recodeflow_missing(details, variable) garbage_rules <- .recodeflow_garbage_rules(var_row) + # Formula-type construction takes precedence over the kind dispatch below + # (#39, Phase A, ADR D1(a)/D6): a non-blank mockFormula on the variable's + # detail rows means it is generated by evaluate_mock_formulas(), regardless + # of what variableType/rType the row otherwise declares. Missing-code and + # garbage settings are reused as-is so postprocess behaves identically to + # any other variable. + mock_formula <- .details_mock_formula(details) + if (!.is_blank(mock_formula)) { + return(mock_spec_formula( + name = variable, + formula = mock_formula, + rtype = rtype, + missing_codes = missing$codes, + missing_proportions = missing$proportions, + garbage_rules = garbage_rules, + provenance = provenance, + model_hint = "native" + )) + } + if (kind == "categorical") { proportions <- extract_proportions(details, variable_name = variable) if (length(proportions$categories) == 0) { stop("Variable '", variable, "' has no valid categorical levels.", call. = FALSE) } + categorical_missing <- .expand_range_missing_codes( + names(proportions$missing), + as.numeric(unlist(proportions$missing, use.names = FALSE)), + variable + ) return(mock_spec_categorical( name = variable, levels = proportions$categories, proportions = proportions$category_proportions, rtype = rtype, - missing_codes = names(proportions$missing), - missing_proportions = as.numeric(unlist(proportions$missing, use.names = FALSE)), + missing_codes = categorical_missing$codes, + missing_proportions = categorical_missing$proportions, garbage_rules = garbage_rules, provenance = provenance, model_hint = "native" @@ -381,7 +475,9 @@ #' This adapter preserves recodeflow semantics instead of treating metadata as a #' generic table. It uses exact role and `databaseStart` token matching, parses #' valid ranges from `recStart`, classifies missing codes from `recEnd` values -#' that begin with `NA::`, preserves categorical levels and proportions, carries +#' that begin with `NA::` (expanding an integer range such as `[997,999]` into +#' its individual codes, with the row's proportion split equally), preserves +#' categorical levels and proportions, carries #' `garbage_*` settings into `garbage_rules`, and stores survival/date fields #' such as `rate`, `shape`, `followup_min`, `followup_max`, and `event_prop` on #' date variables for later backend milestones. @@ -470,6 +566,19 @@ mock_spec_from_recodeflow <- function(variables, if (isTRUE(exclude_derived) && !is.null(variable_details)) { derived <- identify_derived_vars(variables, variable_details) + + # D6 carve-out (#39, Phase A): a DerivedVar::/Func:: variable carrying a + # non-blank mockFormula is intentionally generated by + # evaluate_mock_formulas(), not excluded here. Bare DerivedVar::/Func:: + # rows (no mockFormula) keep today's exclude-and-defer-to-recodeflow + # behaviour unchanged. + has_mock_formula <- vapply(derived, function(v) { + !.is_blank(.details_mock_formula( + .filter_recodeflow_details(variable_details, v, databaseStart) + )) + }, logical(1)) + derived <- derived[!has_mock_formula] + removed <- intersect(variables$variable, derived) if (length(removed) > 0) { message( diff --git a/R/mock_spec_simstudy.R b/R/mock_spec_simstudy.R index 57bac79..236a922 100644 --- a/R/mock_spec_simstudy.R +++ b/R/mock_spec_simstudy.R @@ -128,7 +128,14 @@ #' @noRd .native_only_variables <- function(spec) { - Filter(function(variable) !.simstudy_can_generate(variable), spec$variables) + # Mirror generate_mock_data_native()'s skip: type = "formula" variables have + # no distribution to sample from in either backend. They are computed + # post-baseline by evaluate_mock_formulas(), not by .generate_native_variable() + # (which has no "formula" branch and would stop with an unhelpful "Native + # backend does not support variable type 'formula'"). + Filter(function(variable) { + !.is_formula_variable(variable) && !.simstudy_can_generate(variable) + }, spec$variables) } #' @noRd @@ -137,6 +144,15 @@ return(.empty_native_data(n)) } + if (n == 0) { + # simstudy::genData(0, def) builds its id table as data.table(x = 1:n), + # and 1:0 is c(1, 0) -- two rows -- so formula evaluation stops with + # "Both 'dtSim' and 'n' are set but are of different length". No draws + # happen at zero rows, so the native generators yield the identical typed + # zero-row schema (see the edge-case contract tests). Issue #50. + return(.generate_native_only_baseline(variables, n)) + } + def <- NULL for (variable in variables) { def <- .simstudy_definition(def, variable) @@ -202,6 +218,11 @@ #' `simstudy`. They remain MockData-owned post-processing so both backends share #' the same auditability contract. #' +#' Like [generate_mock_data_native()], this backend skips `type = "formula"` +#' variables — they carry no distribution to sample from. They are computed +#' post-baseline by [evaluate_mock_formulas()], which works over either +#' backend's output. +#' #' @param spec A `mock_spec` object. #' @param n Non-negative whole number of rows to generate. #' @param seed Optional whole-number seed. Generation uses an isolated @@ -209,10 +230,12 @@ #' exit, so output is reproducible for a given seed and package version #' without perturbing the caller's RNG. #' -#' @return A data frame with one column per `mock_spec` variable and `n` rows. +#' @return A data frame with `n` rows and one column per non-formula +#' `mock_spec` variable (`type = "formula"` variables are appended +#' afterwards by [evaluate_mock_formulas()]). #' @family mock generation APIs #' @seealso [generate_mock_data_native()], [postprocess_mock_data()], -#' [mock_spec()] +#' [mock_spec()], [evaluate_mock_formulas()] #' #' @examples #' spec <- mock_continuous("age", range = c(18, 85), rtype = "integer") @@ -229,6 +252,13 @@ generate_mock_data_simstudy <- function(spec, n, seed = NULL) { simstudy_variables <- .simstudy_variables(spec) native_only_variables <- .native_only_variables(spec) + # type = "formula" variables are excluded from both `simstudy_variables` and + # `native_only_variables` above; exclude them here too so the final + # assembly doesn't index `columns` by names that were never generated (see + # generate_mock_data_native()'s identical formula skip). + generated_variable_names <- names(spec$variables)[ + !vapply(spec$variables, .is_formula_variable, logical(1)) + ] .with_mock_seed(seed, { simstudy_data <- .generate_simstudy_baseline(simstudy_variables, n) @@ -237,12 +267,12 @@ generate_mock_data_simstudy <- function(spec, n, seed = NULL) { native_data <- .generate_native_only_baseline(native_only_variables, n) columns <- c(simstudy_data, native_data) - if (length(spec$variables) == 0) { + if (length(generated_variable_names) == 0) { return(.empty_native_data(n)) } as.data.frame( - columns[names(spec$variables)], + columns[generated_variable_names], stringsAsFactors = FALSE, check.names = FALSE ) diff --git a/_pkgdown.yml b/_pkgdown.yml index b3169e3..64be322 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -22,9 +22,11 @@ reference: - mock_continuous - mock_categorical - mock_date + - mock_formula - mock_spec_continuous - mock_spec_categorical - mock_spec_date + - mock_spec_formula - is_mock_spec - validate_mock_spec - mock_spec_model_hints @@ -37,6 +39,7 @@ reference: - mock_spec_from_recodeflow - generate_mock_data_native - generate_mock_data_simstudy + - evaluate_mock_formulas - postprocess_mock_data - title: Main generation functions diff --git a/development/adr/v05-formula-evaluator.md b/development/adr/v05-formula-evaluator.md index 219652d..21139b0 100644 --- a/development/adr/v05-formula-evaluator.md +++ b/development/adr/v05-formula-evaluator.md @@ -1,6 +1,6 @@ # ADR: v0.5 Formula-Derived Variables -**Status**: ACCEPTED 2026-07-07 — Phase A (algebraic formulas), via a `mockFormula` extension column on `variable_details`, evaluated as a post-baseline stage in a restricted sandbox. Any individual decision remains open to maintainer revision before implementation begins. +**Status**: ACCEPTED 2026-07-07 — Phase A (algebraic formulas), via a `mockFormula` extension column on `variable_details`, evaluated as a post-baseline stage in a restricted sandbox. Any individual decision remains open to maintainer revision before implementation begins. Implemented on branch v05-formula-evaluator (Phase A). **Date**: 2026-07-07 **Decision owner**: MockData maintainers **Issue**: #39 @@ -205,6 +205,14 @@ metadata with no `mockFormula` behaves identically to today. exclude). - `Func::` generation remains unsupported (documented), pending Phase B. +**Phase B caveat:** allow-listed base functions can reach R internal-generic S3 +dispatch (e.g. `as.factor` → `as.character` generic), which resolves methods in +the caller's globalenv. This is not exploitable in Phase A (formulas cannot +assign a class to a value — no `structure`/`class<-`/`oldClass` in the +allow-list — and MockData only feeds numeric/factor/character/Date columns to +eval). If Phase B introduces `Func::` dispatch or class-producing operations, +this dispatch path must be re-audited. + **Costs / risks:** - The sandbox allow-list is a maintained surface; too narrow frustrates users, diff --git a/development/adr/v05-seed-contract.md b/development/adr/v05-seed-contract.md index b103d02..b8e8bc3 100644 --- a/development/adr/v05-seed-contract.md +++ b/development/adr/v05-seed-contract.md @@ -1,6 +1,6 @@ # ADR: v0.5 Seed and RNG-Stream Contract -**Status**: ACCEPTED 2026-07-06 — full-bundle contract, co-released with #39 in v0.5. Implementation is a separate ADR-gated plan (not yet written). Any individual decision below remains open to maintainer revision before implementation begins. +**Status**: ACCEPTED 2026-07-06 — full-bundle contract, co-released with #39 in v0.5. Implemented per `development/plans/2026-07-06-seed-contract.md` and merged to `dev` via PR #48 (2026-07-16). Follow-up: #47 (standalone generators). **Date**: drafted 2026-07-03; accepted 2026-07-06 **Decision owner**: MockData maintainers **Issue**: #38 (fulfils #22; resolves #33 item 11) diff --git a/development/plans/2026-07-16-formula-evaluator.md b/development/plans/2026-07-16-formula-evaluator.md new file mode 100644 index 0000000..e56dd30 --- /dev/null +++ b/development/plans/2026-07-16-formula-evaluator.md @@ -0,0 +1,692 @@ +# Formula-Derived Variables Implementation Plan (#39, Phase A) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Variables carrying a `mockFormula` expression are generated by evaluating that expression over already-generated columns — in dependency order, in a restricted sandbox — so mock data can exercise derived-variable pipelines. Algebraic formulas only (Phase A); `Func::` dispatch stays excluded. + +**Architecture:** A third pipeline stage. `generate_mock_data_native()` skips `type = "formula"` variables; a new exported `evaluate_mock_formulas(data, spec, seed = NULL)` computes them post-baseline (topologically ordered among themselves, evaluated in an environment exposing only generated columns + a fixed base-function allow-list, under `.with_mock_seed(stage = "formula")`); `postprocess_mock_data()` then applies missing codes/garbage to all columns including derived ones. Orchestrator order: baseline → formulas → postprocess. Formulas are computed on *clean* baseline values (before contamination), then may receive their own missing codes/garbage — and postprocess's existing missing-column check makes running the stages out of order fail loudly. + +**Tech Stack:** base R (`str2lang`, `all.vars`, `all.names`, `eval` with restricted environments — no new dependency), testthat edition 3, roxygen2. + +**Decision source:** [development/adr/v05-formula-evaluator.md](../adr/v05-formula-evaluator.md) (ACCEPTED 2026-07-07): D1(a) `mockFormula` column on `variable_details`; D2(a) post-baseline stage; D3 restricted sandbox + ratified allow-list; D4 Phase A only; D5 diagnostics marking; D6 `mockFormula` ⇒ generated, bare `DerivedVar::`/`Func::` ⇒ still excluded. + +**Base:** branch `v05-seed-contract` head `0f9bda8` (#38 provides `.with_mock_seed(stage=)` and `.MOCK_STAGES["formula"]`). If PR #48 has merged, branch `v05-formula-evaluator` off `dev` instead — verify `.MOCK_STAGES` exists before starting. + +## Global Constraints + +Inherited from `development/post-v040-development-plan.md` Global Constraints (commit style, fixtures via `system.file()`+skip, CI locale, no check artefacts, review before push). Plus: + +- **Zero seeded-output change for existing specs.** This feature is additive; specs without formula variables must produce byte-identical output. Enforced mechanically by #48's pins in `tests/testthat/test-seed-contract.R`: the frozen `.MOCK_STAGES` equality test and the baseline/postprocess pinned values. If any of those fail during this work, the change is wrong — fix the code, never the pins. +- **Formula variables draw exclusively from the `formula` sub-stream** (`.MOCK_STAGES["formula"]`, index 2 — already reserved). Note: the Phase A allow-list contains no RNG functions, so Phase A formulas are deterministic; the sub-stream wrap is structural (future-proofing), not load-bearing yet. +- **Sandbox is closed:** evaluation must not reach the caller's frame, globalenv, or any function outside the allow-list. `enclos = parent.frame()` (the spike's approach) is prohibited. +- **recodeflow semantics untouched:** `DerivedVar::`/`Func::` meaning unchanged; `mockFormula` is a MockData-extension column; metadata without it behaves identically to today (regression-tested). +- Test suite baseline at plan date: FAIL 0 | WARN 35 | SKIP 11 | PASS 699. Do not exceed WARN/SKIP with new noise. + +## Evidence base (verified 2026-07-16 at `0f9bda8`) + +- Spike machinery to recover: `git show f4f9b41:development/v04-simstudy-spike/prototype.R` — `formula_dependencies()` (`all.vars(str2lang(...))`), `order_spec_by_dependencies()` (progress-loop topological sort; cycle message "Formula dependency cycle or unresolved ordering among: ..."), `validate_formula_referents()`. Its eval (`enclos = parent.frame()`) is NOT recovered — replaced by the D3 sandbox. +- Spec fields `formula` / `depends_on` already exist on `.new_mock_spec_variable()` (`R/mock_spec.R:101,106`), unused. +- Per-variable validator branches on type at `R/mock_spec.R:815/834/852`, else **errors "unsupported type"** at `:855` — a `formula` branch is mandatory or construction-time validation rejects formula specs. +- v0.4 supported-check **already flags any formula-carrying variable as unsupported** (`R/create_mock_data.R:12-18`) → legacy fallback. Task 3 inverts this for `type == "formula"` (and keeps flagging a stray `formula` on a *non*-formula variable). +- Orchestrator pipeline seam: `R/create_mock_data.R:98-102` (`baseline <- generate_mock_data_native(...)` then `postprocess_mock_data(...)`). +- Native dispatch `.generate_native_variable()` at `R/mock_spec_native.R:314-323`; generation loop `columns <- lapply(spec$variables, .generate_native_variable, n = n)` at `:383` — formula vars must be filtered out before this lapply. +- `postprocess_mock_data()` missing-column check (`setdiff(names(spec$variables), names(data))`) means skipping the formula stage fails loudly. `.postprocess_variable()` dispatches on `type == "categorical" || rtype == "integer"` (`R/mock_spec_postprocess.R:154`) — formula variables carry a concrete `rtype`, so missing-code/garbage handling keys off rtype as usual; Task 3 verifies with a test. +- Adapter: `mock_spec_from_recodeflow(..., exclude_derived = TRUE)` excludes `identify_derived_vars()` hits at `R/mock_spec_recodeflow.R:471-472`; kind resolution `.recodeflow_variable_kind(var_row)` at `:117`; per-variable construction `.recodeflow_to_spec_variable(var_row, details, databaseStart)` at `:306`; column access pattern `.row_character(row, col, default)`. +- Docs that will be stale the moment this merges (Task 4): `vignettes/design-philosophy-v04.qmd` "What is deliberately deferred" (formula variables listed as deferred); `vignettes/migrating-from-v03-v04.qmd` fallback-conditions section (names formula variables as a legacy-fallback trigger). + +### Decisions carried from the ADR, resolved here + +- **Direct API:** a dedicated `mock_formula()` constructor (+ `mock_spec_formula()` composable form), not a `formula=` argument on existing constructors — formula variables have no `range`/`levels`/`distribution`, so overloading the other constructors would create nonsense parameter combinations. +- **Diagnostics fields (D5):** `derived = TRUE`, `formula` (string), `depends_on` (character vector) on the variable's `mockdata_diagnostics` entry. +- **`mockFormula` read semantics:** first non-blank `mockFormula` among the variable's `variable_details` rows (a variable's details may span multiple rows; the formula is a variable-level fact stored row-level per ADR). +- **Allow-list:** the ADR's ratified set **plus `c`** (required by `cut(breaks = c(...))`; the ADR authorizes widening on concrete need). + +### Task map + +``` +Task 1: spec layer — constructor, validator branch, allow-list, deps/order/referent checks +Task 2: evaluator — evaluate_mock_formulas(); native backend skips formula vars +Task 3: wiring — adapter mockFormula + D6 carve-out; orchestrator stage; diagnostics; e2e +Task 4: docs — NEWS, roxygen family, stale-vignette fixes, ADR status note +``` + +--- + +### Task 1: Formula spec layer + +**Files:** +- Modify: `R/mock_spec.R` (constructors near `mock_categorical()`; validator continuous/categorical/date chain at :815-855; `validate_mock_spec()` for spec-level checks) +- Create: `R/mock_spec_formula.R` (allow-list, dependency/order/referent helpers — the evaluator itself is Task 2) +- Test: `tests/testthat/test-formula-vars.R` (new) + +**Interfaces:** +- Produces (consumed by Tasks 2–3): + - `mock_formula(name, formula, rtype = "double", missing_codes = numeric(0), missing_proportions = numeric(0), garbage_rules = list(), provenance = NULL, model_hint = "auto")` → validated one-variable `mock_spec`; `mock_spec_formula(...)` → `mock_spec_variable` with `type = "formula"`. + - `.formula_allowlist` (character vector), `.formula_dependencies(variable)`, `.formula_function_symbols(variable)`, `.order_formula_variables(spec)` (returns ordered names of formula variables only; cycle error), `.validate_formula_referents(spec)` (unknown-referent + disallowed-symbol errors). + +- [x] **Step 1: Write the failing tests** + +```r +# tests/testthat/test-formula-vars.R +# Formula-derived variables (#39, Phase A). ADR: development/adr/v05-formula-evaluator.md + +test_that("mock_formula constructs a validated formula variable", { + spec <- mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_continuous("weight", range = c(45, 150)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ) + expect_true(validate_mock_spec(spec)$valid) + expect_identical(spec$variables$bmi$type, "formula") + expect_identical(spec$variables$bmi$formula, "weight / (height^2)") + expect_setequal(spec$variables$bmi$depends_on, c("weight", "height")) +}) + +test_that("a formula referencing an unknown variable fails validation", { + expect_error( + mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ), + "references unknown variable" + ) +}) + +test_that("a disallowed function symbol fails validation before generation", { + expect_error( + mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "system('echo pwned') + x") + ), + "not permitted in mockFormula" + ) +}) + +test_that("a dependency cycle among formula variables errors clearly", { + expect_error( + mock_spec( + mock_spec_formula("a", formula = "b + 1"), + mock_spec_formula("b", formula = "a + 1") + ), + "dependency cycle" + ) +}) + +test_that("an unparseable formula errors at validation", { + expect_error( + mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2") + ), + "could not be parsed" + ) +}) +``` + +- [x] **Step 2: Run to verify failure** — `Rscript -e 'devtools::load_all(quiet = TRUE); testthat::test_file("tests/testthat/test-formula-vars.R")'`. Expected: FAIL, `could not find function "mock_spec_formula"`. + +- [x] **Step 3: Create `R/mock_spec_formula.R` helpers** + +```r +# ============================================================================== +# Formula-derived variables (#39, Phase A) — spec-side helpers. +# Evaluation itself lives in evaluate_mock_formulas() (same file, Task 2). +# ADR: development/adr/v05-formula-evaluator.md +# ============================================================================== + +# Base functions/operators a mockFormula may call. Deliberately closed and +# RNG-free: Phase A formulas are deterministic transformations of generated +# columns. Widen only on concrete need (ADR D3). +.formula_allowlist <- c( + "+", "-", "*", "/", "^", "%%", "%/%", "(", + "<", "<=", ">", ">=", "==", "!=", "&", "|", "!", + "ifelse", "pmin", "pmax", "min", "max", "abs", "round", "floor", + "ceiling", "sqrt", "exp", "log", "log10", "sum", "mean", "cut", + "as.numeric", "as.integer", "as.factor", "factor", "c" +) + +#' @noRd +.parse_mock_formula <- function(variable) { + tryCatch( + str2lang(variable$formula), + error = function(e) { + stop( + "Formula for variable '", variable$name, "' could not be parsed: ", + conditionMessage(e), + call. = FALSE + ) + } + ) +} + +#' @noRd +.is_formula_variable <- function(variable) { + identical(variable$type, "formula") +} + +#' @noRd +.formula_dependencies <- function(variable) { + if (!.is_formula_variable(variable)) { + return(character(0)) + } + all.vars(.parse_mock_formula(variable)) +} + +#' @noRd +.formula_function_symbols <- function(variable) { + expr <- .parse_mock_formula(variable) + setdiff(all.names(expr), all.vars(expr)) +} + +#' @noRd +.validate_formula_referents <- function(spec) { + spec_names <- names(spec$variables) + for (variable in spec$variables) { + if (!.is_formula_variable(variable)) next + missing <- setdiff(.formula_dependencies(variable), spec_names) + if (length(missing) > 0) { + stop( + "Formula for variable '", variable$name, + "' references unknown variable(s): ", + paste(missing, collapse = ", "), + call. = FALSE + ) + } + disallowed <- setdiff(.formula_function_symbols(variable), .formula_allowlist) + if (length(disallowed) > 0) { + stop( + "Formula for variable '", variable$name, + "' uses function(s) not permitted in mockFormula expressions: ", + paste(disallowed, collapse = ", "), + call. = FALSE + ) + } + } + invisible(TRUE) +} + +#' @noRd +.order_formula_variables <- function(spec) { + formula_names <- names(spec$variables)[ + vapply(spec$variables, .is_formula_variable, logical(1)) + ] + remaining <- formula_names + ordered <- character(0) + while (length(remaining) > 0) { + progressed <- FALSE + for (name in remaining) { + deps <- intersect(.formula_dependencies(spec$variables[[name]]), formula_names) + if (all(deps %in% ordered)) { + ordered <- c(ordered, name) + remaining <- setdiff(remaining, name) + progressed <- TRUE + } + } + if (!progressed) { + stop( + "Formula dependency cycle or unresolved ordering among: ", + paste(remaining, collapse = ", "), + call. = FALSE + ) + } + } + ordered +} +``` + +- [x] **Step 4: Constructors in `R/mock_spec.R`** + +`mock_spec_formula()` (place after `mock_spec_date()`), populating the existing `formula`/`depends_on` fields via `.new_mock_spec_variable()`: + +```r +#' Create a formula-derived variable specification +#' +#' `mock_spec_formula()` describes a variable computed from other generated +#' variables by evaluating an algebraic expression (a `mockFormula`), rather +#' than sampled from a distribution. Evaluation happens in a restricted +#' environment exposing only the generated columns and a fixed allow-list of +#' base functions; see [evaluate_mock_formulas()]. +#' +#' @param name Variable name. +#' @param formula Character scalar. An algebraic expression over other +#' variable names, e.g. `"weight / (height^2)"`. +#' @param rtype R output type for the computed column. Defaults to `"double"`. +#' @param missing_codes,missing_proportions,garbage_rules,provenance,model_hint +#' As for other variable specifications; applied by post-processing. +#' +#' @return A `mock_spec_variable` object of type `"formula"`. +#' @family mock specification APIs +#' @seealso [mock_formula()], [evaluate_mock_formulas()] +#' @export +mock_spec_formula <- function(name, + formula, + rtype = "double", + missing_codes = numeric(0), + missing_proportions = numeric(0), + garbage_rules = list(), + provenance = "direct", + model_hint = "auto") { + variable <- .new_mock_spec_variable( + name = name, + type = "formula", + rtype = rtype, + formula = formula, + missing_codes = missing_codes, + missing_proportions = missing_proportions, + garbage_rules = garbage_rules, + provenance = provenance, + model_hint = model_hint + ) + # Parse failures are reported by validate_mock_spec(), not the constructor, + # matching the sibling constructors' defer-all-validation contract. + # depends_on is computed once here at construction time; it is not + # re-synced if `formula` is mutated afterwards. + variable$depends_on <- tryCatch( + .formula_dependencies(variable), + error = function(e) character(0) + ) + variable +} +``` + +`mock_formula()` (simple direct API, mirroring `mock_continuous()`'s shape: wraps `mock_spec_formula()` in `mock_spec(...)` with `.direct_api_provenance("mock_formula", provenance)`). Follow the `mock_continuous()` pattern at `R/mock_spec.R:287` exactly, forwarding the same parameters. + +- [x] **Step 5: Validator branches** + +Per-variable (`R/mock_spec.R`, extend the type chain before the `:855` unsupported-type error): + +```r + } else if (variable$type == "formula") { + f <- variable$formula + if (is.null(f) || !is.character(f) || length(f) != 1 || is.na(f) || + trimws(f) == "") { + errors <- c(errors, paste0( + "Variable '", variable$name, + "' formula type requires a non-empty character formula." + )) + } else { + parse_error <- tryCatch({ str2lang(f); NULL }, error = function(e) conditionMessage(e)) + if (!is.null(parse_error)) { + errors <- c(errors, paste0( + "Variable '", variable$name, "' formula could not be parsed: ", parse_error + )) + } + } + if (!variable$rtype %in% c("double", "numeric", "integer", "factor", "character", "logical")) { + errors <- c(errors, paste0( + "Variable '", variable$name, "' has unsupported formula rType '", variable$rtype, "'." + )) + } + } +``` + +Spec-level (`validate_mock_spec()`, after per-variable validation succeeds): call `.validate_formula_referents(spec)` and `.order_formula_variables(spec)` inside a tryCatch that converts their `stop()`s into validation errors in the same style as existing spec-level checks — read how `validate_mock_spec()` accumulates errors and match it. Constructor-time validation (`mock_spec(validate = TRUE)`) then rejects unknown referents, disallowed symbols, and cycles at construction, which is what the Step 1 tests assert. + +- [x] **Step 6: Run the test file** — expected all PASS. **Bounded contingency:** if construction is *lazy* for spec-level checks (errors only via explicit `validate_mock_spec()`), check how `mock_spec()` invokes validation (`R/mock_spec.R:239` area, `validate = TRUE` default → strict) — the seed/exponential work confirmed construction validates eagerly, so eager is expected; if a test still fails, STOP and report rather than weakening the test. + +- [x] **Step 7: `devtools::document()`; full suite; commit** + +Run full suite — expected: baseline counts + new passes, no new WARN/SKIP. + +```bash +git add R/mock_spec.R R/mock_spec_formula.R tests/testthat/test-formula-vars.R man/ NAMESPACE +git commit -m "Add formula variable type to the mock_spec layer (#39) + +mock_formula()/mock_spec_formula() constructors populate the reserved +formula/depends_on fields; validation rejects unparseable formulas, +unknown referents, disallowed function symbols, and dependency cycles +at construction, using the allow-list and topological ordering +recovered from the v0.4 spike." +``` + +### Task 2: Evaluator + native-backend skip + +**Files:** +- Modify: `R/mock_spec_formula.R` (add `evaluate_mock_formulas()` + eval env helpers) +- Modify: `R/mock_spec_native.R` (`generate_mock_data_native()` — filter formula vars from the `:383` lapply) +- Test: `tests/testthat/test-formula-vars.R` (extend) + +**Interfaces:** +- Consumes: Task 1 helpers; `.with_mock_seed(seed, expr, stage)` and `.MOCK_STAGES["formula"]` from #38. +- Produces: exported `evaluate_mock_formulas(data, spec, seed = NULL)` → data frame with formula columns appended (order of existing columns unchanged; no-op returning `data` when the spec has no formula variables). Consumed by Task 3's orchestrator. + +- [x] **Step 1: Write the failing tests** + +```r +test_that("evaluate_mock_formulas computes algebraic derivations correctly", { + spec <- mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_continuous("weight", range = c(45, 150)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ) + baseline <- generate_mock_data_native(spec, n = 50, seed = 11) + expect_false("bmi" %in% names(baseline)) # native skips formula vars + result <- evaluate_mock_formulas(baseline, spec, seed = 11) + expect_identical(result$bmi, baseline$weight / (baseline$height^2)) + expect_identical(result$height, baseline$height) # inputs untouched +}) + +test_that("formula-of-a-formula evaluates in dependency order", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 10)), + mock_spec_formula("obese_flag", formula = "ifelse(y > 15, 1, 0)", rtype = "integer"), + mock_spec_formula("y", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 20, seed = 5) + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(result$y, baseline$x * 2) + expect_identical(result$obese_flag, as.integer(ifelse(baseline$x * 2 > 15, 1L, 0L))) +}) + +test_that("evaluation cannot reach the caller's environment", { + leaky <- 999 + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x + leaky") + ) + # 'leaky' is not a spec variable -> referent validation must reject it at + # construction; the sandbox property is enforced before eval ever runs. + expect_error(validate_mock_spec(spec), NA) # if construction was lazy +}, ) +# NOTE to implementer: the construction call above should already error under +# eager validation ("references unknown variable"), making the test body as +# written unreachable. Restructure to expect_error(mock_spec(...), "references +# unknown variable") if construction is eager (expected). Keep ONE test that +# pins the sandbox at eval level too: + +test_that("eval environment exposes only columns and allow-listed functions", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "sqrt(x)") + ) + baseline <- generate_mock_data_native(spec, n = 5, seed = 3) + env_probe <- evaluate_mock_formulas(baseline, spec) + expect_identical(env_probe$z, sqrt(baseline$x)) +}) + +test_that("evaluate_mock_formulas is a no-op without formula variables", { + spec <- mock_continuous("x", range = c(0, 1)) + baseline <- generate_mock_data_native(spec, n = 5, seed = 3) + expect_identical(evaluate_mock_formulas(baseline, spec), baseline) +}) + +test_that("evaluate_mock_formulas handles n = 0", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 0, seed = 3) + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(nrow(result), 0L) + expect_true("z" %in% names(result)) +}) + +test_that("formula stage leaves the caller's RNG untouched and is deterministic", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 10, seed = 7) + set.seed(123); before <- .Random.seed + a <- evaluate_mock_formulas(baseline, spec, seed = 7) + expect_identical(.Random.seed, before) + expect_identical(a, evaluate_mock_formulas(baseline, spec, seed = 7)) +}) + +test_that("rtype coercion applies to formula outputs", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 9)), + mock_spec_formula("band", formula = "ifelse(x > 5, 'high', 'low')", rtype = "factor") + ) + baseline <- generate_mock_data_native(spec, n = 20, seed = 9) + result <- evaluate_mock_formulas(baseline, spec) + expect_s3_class(result$band, "factor") +}) +``` + +(Clean up the sandbox-test scaffolding per the NOTE — the plan expects eager construction validation, so the "leaky" test becomes `expect_error(mock_spec(...), "references unknown variable")`.) + +- [x] **Step 2: Run — verify failures** (`could not find function "evaluate_mock_formulas"`; the native-skip expectation fails because `.generate_native_variable()` stops on type "formula"). + +- [x] **Step 3: Native skip** + +In `generate_mock_data_native()` (`R/mock_spec_native.R`, around `:383`), before the lapply, filter: + +```r + generated <- spec$variables[ + !vapply(spec$variables, .is_formula_variable, logical(1)) + ] + columns <- lapply(generated, .generate_native_variable, n = n) +``` + +(Adapt names to the actual local code; the empty-formula-only-spec case must still return an `n`-row 0-column frame via the existing `.empty_native_data(n)` path — check how the current code assembles `columns` into a data frame and preserve it.) Document the skip in the function's roxygen `@details`: formula variables are produced by `evaluate_mock_formulas()`, not the backend. + +- [x] **Step 4: The evaluator** (append to `R/mock_spec_formula.R`) + +```r +#' @noRd +.formula_parent_env <- function() { + env <- new.env(parent = emptyenv()) + for (fn in .formula_allowlist) { + env[[fn]] <- get(fn, envir = baseNamespace()) + } + env +} + +#' Evaluate formula-derived variables over generated data +#' +#' Computes each `type = "formula"` variable in `spec` by evaluating its +#' expression over the columns of `data`, in dependency order. Evaluation runs +#' in a restricted environment exposing only the data columns and a fixed +#' allow-list of base functions — formulas cannot reach the caller's +#' environment. Any randomness would draw from the isolated `formula` +#' sub-stream (see the v0.5 seed contract), though the Phase A allow-list is +#' RNG-free, so evaluation is deterministic given `data`. +#' +#' @param data Data frame of generated baseline values (from +#' [generate_mock_data_native()] or [generate_mock_data_simstudy()]). +#' @param spec A `mock_spec`. Non-formula variables must already be columns of +#' `data`. +#' @param seed Optional whole-number seed; reserved for future formulas with +#' random components. The caller's RNG state and kind are restored on exit. +#' +#' @return `data` with one appended column per formula variable. Returns +#' `data` unchanged if the spec has no formula variables. +#' @seealso [mock_formula()], [postprocess_mock_data()] +#' @export +evaluate_mock_formulas <- function(data, spec, seed = NULL) { + if (!is.data.frame(data)) { + stop("data must be a data frame.", call. = FALSE) + } + # Strict-validate first, exactly as postprocess_mock_data() does (Task 2 + # review fix): the accumulate-and-skip behaviour inside + # .order_formula_variables()/.validate_formula_referents() is correct for + # validate_mock_spec()'s multi-error reporting, but it means an unparseable, + # cyclic, or invalid-referent formula would otherwise be silently excluded + # from `ordered` below rather than reported. Strict validation already runs + # .validate_formula_referents()/.order_formula_variables(), so the direct + # .validate_formula_referents() call that used to live below is redundant + # and has been removed. + validate_mock_spec(spec, n = nrow(data), strict = TRUE) + + ordered <- .order_formula_variables(spec) + if (length(ordered) == 0) { + return(data) + } + + non_formula <- setdiff(names(spec$variables), names(spec$variables)[ + vapply(spec$variables, .is_formula_variable, logical(1)) + ]) + missing_inputs <- setdiff(non_formula, names(data)) + if (length(missing_inputs) > 0) { + stop( + "data is missing generated column(s) required by formulas: ", + paste(missing_inputs, collapse = ", "), + call. = FALSE + ) + } + + .with_mock_seed(seed, stage = "formula", { + fn_env <- .formula_parent_env() + for (name in ordered) { + variable <- spec$variables[[name]] + values <- eval( + .parse_mock_formula(variable), + envir = as.list(data), + enclos = fn_env + ) + if (length(values) == 1 && nrow(data) != 1) { + values <- rep(values, nrow(data)) # scalar formulas broadcast + } + if (length(values) != nrow(data)) { + stop( + "Formula for variable '", name, "' returned length ", length(values), + ", expected ", nrow(data), ".", + call. = FALSE + ) + } + data[[name]] <- .coerce_formula_rtype(values, variable$rtype, name) + } + data + }) +} + +#' @noRd +.coerce_formula_rtype <- function(values, rtype, variable_name) { + switch(rtype, + double = , + numeric = as.numeric(values), + integer = as.integer(values), + factor = as.factor(values), + character = as.character(values), + logical = as.logical(values), + stop( + "Variable '", variable_name, "' has unsupported formula rType '", + rtype, "'.", + call. = FALSE + ) + ) +} +``` + +Note on `envir = as.list(data)`: eval with a list envir + `enclos` gives column lookup first, then the allow-list env, whose parent is `emptyenv()` — nothing else is reachable. Verify with the sandbox tests; if `as.list()` + enclos misbehaves for `nrow(data) == 0` columns, use `list2env(as.list(data), parent = fn_env)` and drop `enclos` — either satisfies the tests; pick the one that passes and note which in the report. + +- [x] **Step 5: Run tests → PASS; full suite; commit** (message: "Add evaluate_mock_formulas() with sandboxed post-baseline evaluation (#39)" + body noting native backend skips formula variables and the sandbox contract). + +### Task 3: Adapter + orchestrator wiring, diagnostics, end-to-end + +**Files:** +- Modify: `R/mock_spec_recodeflow.R` (kind resolution `:117`; `.recodeflow_to_spec_variable()` `:306`; `exclude_derived` block `:471-472`) +- Modify: `R/create_mock_data.R` (unsupported-check `:12-18`; pipeline `:98-102`) +- Modify: `R/mock_spec_postprocess.R` (`.postprocess_empty_diagnostics()` — D5 fields) +- Test: `tests/testthat/test-formula-vars.R` (extend) + +**Interfaces:** +- Consumes: Tasks 1–2. Produces: `create_mock_data()` end-to-end support; `mockFormula` metadata column. + +- [x] **Step 1: Write the failing tests** (model in-memory metadata on the recodeflow-exponential integration test in `tests/testthat/test-native-exponential.R` — reuse its exact column structure, adding a `mockFormula` column on the details rows): + +```r +test_that("mockFormula metadata generates a derived column end-to-end", { + # variables: height, weight (continuous) + bmi (Continuous rType double, + # detail row carrying DerivedVar:: recStart AND a mockFormula) + # -> bmi must be GENERATED (D6 carve-out), on the v0.4 path, with + # diagnostics marking it derived. + + result <- suppressMessages(create_mock_data("study", variables, variable_details, n = 30, seed = 4)) + expect_true("bmi" %in% names(result)) + diag <- attr(result, "mockdata_diagnostics") + expect_false(is.null(diag)) # v0.4 path, not legacy + expect_true(isTRUE(diag$variables$bmi$derived)) + expect_setequal(diag$variables$bmi$depends_on, c("height", "weight")) +}) + +test_that("DerivedVar rows without mockFormula stay excluded (D6, backward compat)", { + + result <- suppressMessages(create_mock_data("study", variables, variable_details, n = 10, seed = 4)) + expect_false("bmi" %in% names(result)) +}) + +test_that("formula variables receive their own missing codes in postprocess", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 9)), + mock_spec_formula("z", formula = "x * 2", + missing_codes = -99, missing_proportions = 0.3) + ) + baseline <- generate_mock_data_native(spec, n = 200, seed = 8) + staged <- evaluate_mock_formulas(baseline, spec, seed = 8) + result <- postprocess_mock_data(staged, spec, seed = 8) + expect_true(any(result$z == -99)) + expect_true(all(result$z[result$z != -99] == staged$z[result$z != -99])) +}) + +test_that("existing formula-free seeded output is unchanged by the formula stage", { + # The #48 pinned-reference tests already enforce this mechanically; this + # test additionally pins the orchestrator route: a formula-free spec must + # not pass through evaluate_mock_formulas' seeded wrapper at all. + spec <- mock_continuous("x", range = c(0, 1)) + baseline <- generate_mock_data_native(spec, n = 3, seed = 20260706) + expect_identical(evaluate_mock_formulas(baseline, spec, seed = 20260706), baseline) +}) +``` + +- [x] **Step 2: Adapter changes** (`R/mock_spec_recodeflow.R`): + 1. A helper `.details_mock_formula(details)` returning the first non-blank `mockFormula` among the variable's detail rows ("" default; column may be absent). + 2. In `.recodeflow_to_spec_variable()`: if `.details_mock_formula(details)` is non-blank, construct via `mock_spec_formula(name, formula = , rtype = , missing_codes/garbage from the existing extraction)` instead of the kind-dispatched constructor. The variable's `variableType` may say Continuous — the formula type takes precedence; reuse the row's already-extracted missing-code/garbage settings so postprocess behaviour matches other variables. + 3. The `exclude_derived` block (`:471-472`): filter the `identify_derived_vars()` hits to exclude only those **without** a non-blank `mockFormula` in their detail rows (D6 carve-out). Keep `exclude_derived = TRUE` default semantics otherwise. The `has_mock_formula` check must use the same databaseStart-filtered details view as construction (`.filter_recodeflow_details()`), never the raw `variable == v` subset — otherwise a variable with a mockFormula in one cycle but not another gets kept for a cycle where construction can't find a formula, crashing generation instead of staying excluded (#39 review finding). + +- [x] **Step 3: Orchestrator changes** (`R/create_mock_data.R`): + 1. Unsupported-check (`:12-18`): replace the blanket `has_formula → TRUE` with: `if (variable$type == "formula") return(FALSE)` (supported), and keep `has_formula && type != "formula"` → TRUE (a stray formula on a non-formula variable remains a fallback trigger — defensive). + 2. Pipeline (`:98-102`): insert the stage — + +```r + baseline <- generate_mock_data_native(spec, n = n, seed = seed) + staged <- evaluate_mock_formulas(baseline, spec, seed = seed) + postprocess_mock_data(staged, spec, seed = seed) +``` + + (Update the comment above it to name all three sub-streams.) + +- [x] **Step 4: Diagnostics (D5)** — in `.postprocess_empty_diagnostics()` (`R/mock_spec_postprocess.R`), when building each variable's entry, add for formula variables: `derived = TRUE`, `formula = variable$formula`, `depends_on = variable$depends_on`. Read the function first and match its list-building style exactly. + +- [x] **Step 5: Run tests → PASS. Full suite — the #48 stage-index and pinned-value tests MUST still pass untouched (they prove no output shift). `devtools::check()` clean. Commit** ("Wire mockFormula through the recodeflow adapter and orchestrator (#39)" + body covering the D6 carve-out, the unsupported-check inversion, diagnostics fields). + +### Task 4: Docs, NEWS, stale-claims sweep + +**Files:** +- Modify: `NEWS.md`; `R/` roxygen touch-ups from review; `vignettes/design-philosophy-v04.qmd`; `vignettes/migrating-from-v03-v04.qmd`; `development/adr/v05-formula-evaluator.md` (status line) + +- [x] **Step 1: NEWS** (feature bullet under the development-version heading, NOT in the breaking-change section — this is additive): + +```markdown +## Formula-derived variables (#39, Phase A) + +- Variables can now be generated from algebraic expressions over other + generated variables via a new `mockFormula` column in `variable_details` + (e.g. `weight / (height^2)`), or the new `mock_formula()` direct API. + Formulas are evaluated in dependency order by `evaluate_mock_formulas()`, + in a restricted environment exposing only the generated columns and a + fixed allow-list of base functions. `DerivedVar::`/`Func::` semantics are + unchanged: derived variables without a `mockFormula` remain excluded from + generation, and `Func::` dispatch is not yet supported. +``` + +- [x] **Step 2: Stale-claims sweep** (the #45/#48 reviews both caught rendered-false docs — do not repeat that): + - `vignettes/design-philosophy-v04.qmd` "What is deliberately deferred": formula variables are no longer fully deferred — rewrite that sentence to say algebraic `mockFormula` generation landed in v0.5 (Phase A) while `Func::` dispatch and a general evaluator remain deferred. + - `vignettes/migrating-from-v03-v04.qmd`: the fallback-conditions section names formula variables as a legacy-fallback trigger — verify and update: variables with `mockFormula` now route natively; a stray `formula` field on a non-formula variable still falls back. Run `quarto render vignettes/migrating-from-v03-v04.qmd --to html` and confirm no chunk renders a claim the code contradicts (delete generated output after). + - Grep vignettes + README for `mockFormula`, "formula", "derived" claims: `grep -rn "formula" vignettes/*.qmd README.md | grep -iv "mockFormula"` — inspect hits that assert formulas are unsupported/deferred; fix only genuinely contradicted statements. + - `development/adr/v05-formula-evaluator.md`: append to the Status line: "Implemented in PR # (Phase A)." + +- [x] **Step 3: `devtools::document()`; full suite; `devtools::check()`; commit** ("Document formula-derived variables and refresh deferred-work claims (#39)"). + +--- + +## Self-review (performed at plan date) + +- **ADR coverage:** D1(a) Task 3 Step 2; D2(a) Tasks 2–3 (post-baseline stage, orchestrator order baseline→formula→postprocess); D3 Task 1 allow-list + Task 2 restricted env (+ sandbox tests); D4 Phase A only (no Func:: anywhere; D6 keeps Func:: rows excluded); D5 Task 3 Step 4; D6 Task 3 Steps 1–2. ADR test-strategy items 1–6 all appear across Task 1/2/3 tests. +- **No-break guarantee:** enforced three ways — the #48 frozen-index and pinned-value tests (must pass untouched), the Task 3 formula-free no-op test, and the native skip changing nothing for formula-free specs. +- **Placeholder scan:** Task 3 Step 1's metadata fixtures are directed to copy the existing integration test's exact column structure (the #37 review taught that fixture shapes must come from the suite, not be invented) — the `` markers name the source pattern and the exact distinguishing values; all other code is complete. Two bounded contingencies (eager-vs-lazy validation in Task 1 Step 6; `as.list`+enclos vs `list2env` in Task 2 Step 4) give both branches concretely. +- **Type consistency:** `type = "formula"`, `.is_formula_variable()`, `.formula_allowlist`, `evaluate_mock_formulas(data, spec, seed)`, diagnostics fields `derived`/`formula`/`depends_on` used identically across tasks; `stage = "formula"` matches `.MOCK_STAGES` from #38. +- **Known risk carried:** `.postprocess_variable()`'s type dispatch (`:154`) was only inspected, not executed, for formula variables — Task 3's missing-codes test exercises it; if it misroutes (e.g., factor formula var), that surfaces there, in-task. diff --git a/development/post-v040-development-plan.md b/development/post-v040-development-plan.md index d4794da..7e28681 100644 --- a/development/post-v040-development-plan.md +++ b/development/post-v040-development-plan.md @@ -542,26 +542,26 @@ Tasks 5 and 6 **must land in the same minor release**: both change seeded output **Deliverable:** `development/adr/v05-seed-contract.md`, approved by the maintainer. -- [ ] **Step 1: Draft the ADR** with these sections and decision points: +- [x] **Step 1: Draft the ADR** with these sections and decision points: - **Context:** three regimes (legacy global `set.seed` at `R/create_mock_data.R:325-327`; v0.4 `seed`/`seed + 1L` at `:85-90`; simstudy backend's own handling); fallback stream-switch is verbose-gated (#33 item 11). - **Options:** (a) document the status quo; (b) L'Ecuyer-CMRG sub-streams via `parallel::nextRNGStream()`, one stream per stage; (c) hash-derived per-stage (or per-variable) integer seeds. **Recommendation: (b) at stage granularity** — principled, base-R only, and per-variable granularity is YAGNI until parallel generation is on the table. - **Guarantee scope:** same seed + same spec + same package version ⇒ identical output; stability across minor versions explicitly *not* guaranteed when NEWS declares a break. - **Migration:** single break, coordinated with Task 6; pinned-value regression tests (fulfils #22); decide whether the fallback stream-switch message becomes always-on (#33 item 11). -- [ ] **Step 2: Maintainer review — STOP until approved.** -- [ ] **Step 3: Write the implementation plan** (superpowers:writing-plans → `development/plans/`) and execute. +- [x] **Step 2: Maintainer review — STOP until approved.** +- [x] **Step 3: Write the implementation plan** (superpowers:writing-plans → `development/plans/`) and execute. ### Task 6: Formula-evaluator ADR + implementation (#39) **Deliverable:** `development/adr/v05-formula-evaluator.md`, approved; then phased implementation. -- [ ] **Step 1: Recover the spike:** `git show f4f9b41:development/v04-simstudy-spike/prototype.R > /tmp/spike-prototype.R` and read it. It contains `formula_dependencies()` (`all.vars(str2lang(...))`), topological ordering with cycle detection ("Formula dependency cycle or unresolved ordering among: ..."), and `validate_formula_referents()`. -- [ ] **Step 2: Draft the ADR** with these decision points: +- [x] **Step 1: Recover the spike:** `git show f4f9b41:development/v04-simstudy-spike/prototype.R > /tmp/spike-prototype.R` and read it. It contains `formula_dependencies()` (`all.vars(str2lang(...))`), topological ordering with cycle detection ("Formula dependency cycle or unresolved ordering among: ..."), and `validate_formula_referents()`. +- [x] **Step 2: Draft the ADR** with these decision points: - **Syntax entry:** (a) new MockData-extension column (e.g. `mockFormula`) vs (b) reuse `variableStart` `DerivedVar::`/`Func::`. **Recommendation: (a)** — cchsflow/chmsflow are CRAN downstreams; do not overload shared recodeflow semantics. `identify_derived_vars()` continues to detect-and-skip `DerivedVar::` entries that carry no `mockFormula`. - **Evaluation environment:** restricted env containing only generated columns + an enumerated whitelist of base math/logic functions (`+ - * / ^ %% < <= > >= == != & | ! ifelse pmin pmax log exp sqrt abs round`); no filesystem, network, or global lookup. List the exact whitelist in the ADR. - **Phasing:** Phase A — algebraic formulas over generated columns (spec fields `formula`/`depends_on` already exist, `R/mock_spec.R:94-145`). Phase B — `Func::` dispatch to the consuming package's namespace (define lookup order and failure semantics). - **Diagnostics:** derived columns get a `derived = TRUE` entry in `mockdata_diagnostics` with their dependency list. -- [ ] **Step 3: Maintainer review — STOP until approved.** -- [ ] **Step 4: Implementation plan + execution.** Required tests: referent validation errors, cycle detection message, deterministic ordering, seed reproducibility under the Task 5 contract. +- [x] **Step 3: Maintainer review — STOP until approved.** +- [x] **Step 4: Implementation plan + execution.** Required tests: referent validation errors, cycle detection message, deterministic ordering, seed reproducibility under the Task 5 contract. ### Task 7: Survival pairs in batch generation (#40) diff --git a/inst/extdata/minimal-example/variables.csv b/inst/extdata/minimal-example/variables.csv index 9e5ce85..c8f2dbf 100644 --- a/inst/extdata/minimal-example/variables.csv +++ b/inst/extdata/minimal-example/variables.csv @@ -1,8 +1,8 @@ "uid","variable","label","variableType","rType","role","position","seed","garbage_low_prop","garbage_low_range","garbage_high_prop","garbage_high_range","distribution","mean","sd","rate","shape","followup_min","followup_max","event_prop","sourceFormat","mockDataVersion","mockDataLastUpdated","mockDataVersionNotes" "cchsflow_v0001","age","Age in years","Continuous","integer","enabled,predictor,table1",10,10,,"[;]",,"[;]","normal",50,15,,,,,,"","1.0.0","2025-11-09","Normal distribution (mean=50, sd=15)" "cchsflow_v0002","smoking","Smoking status","Categorical","factor","enabled,predictor,table1",20,20,,"",,"","",,,,,,,,"","1.0.0","2025-11-09","Categorical variable with proportions" -"cchsflow_v0003","BMI","Body mass index","Continuous","double","enabled,outcome,table1",30,30,0.02,"[-10;15])",0.01,"[60;150]","normal",27.5,5.2,,,,,,"","1.0.0","2025-11-09","Normal distribution with two-sided contamination" -"cchsflow_v0004","height","Height in meters","Continuous","double","enabled,predictor",40,40,1,"[0;1.4)",0.01,"(2.1;inf]","normal",1.7,0.1,,,,,,"","1.0.0","2025-11-13","Height for BMI calculation" +"cchsflow_v0003","BMI","Body mass index","Continuous","double","enabled,outcome,table1",30,30,0.02,"[-10;15]",0.01,"[60;150]","normal",27.5,5.2,,,,,,"","1.0.0","2025-11-09","Normal distribution with two-sided contamination" +"cchsflow_v0004","height","Height in meters","Continuous","double","enabled,predictor",40,40,0.01,"[0;1.4)",0.01,"(2.1;inf]","normal",1.7,0.1,,,,,,"","1.0.0","2025-11-13","Height for BMI calculation" "cchsflow_v0005","weight","Weight in kilograms","Continuous","double","enabled,predictor",50,50,,"[;]",,"[;]","normal",75,15,,,,,,"","1.0.0","2025-11-13","Weight for BMI calculation" "cchsflow_v0006","BMI_derived","BMI calculated from height and weight","Continuous","double","enabled,outcome,table1",60,60,,"[;]",,"[;]","",,,,,,,,"","1.0.0","2025-11-13","Derived variable: BMI = weight / (height^2)" "ices_v01","interview_date","Interview date (cohort entry)","Continuous","date","enabled,outcome",70,70,0,"[;]",0,"[;]","uniform",,,,,,,,"analysis","1.0.0","2025-11-09","Uniform date range (cohort entry)" diff --git a/renv.lock b/renv.lock index e6a6d32..14e588b 100644 --- a/renv.lock +++ b/renv.lock @@ -9,6 +9,23 @@ ] }, "Packages": { + "BH": { + "Package": "BH", + "Version": "1.90.0-1", + "Source": "Repository", + "Type": "Package", + "Title": "Boost C++ Header Files", + "Date": "2025-12-13", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"John W.\", \"Emerson\", role = \"aut\"), person(\"Michael J.\", \"Kane\", role = \"aut\", comment = c(ORCID = \"0000-0003-1899-6662\")))", + "Description": "Boost provides free peer-reviewed portable C++ source libraries. A large part of Boost is provided as C++ template code which is resolved entirely at compile-time without linking. This package aims to provide the most useful subset of Boost libraries for template use among CRAN packages. By placing these libraries in this package, we offer a more efficient distribution system for CRAN as replication of this code in the sources of other packages is avoided. As of release 1.84.0-0, the following Boost libraries are included: 'accumulators' 'algorithm' 'align' 'any' 'atomic' 'beast' 'bimap' 'bind' 'circular_buffer' 'compute' 'concept' 'config' 'container' 'date_time' 'detail' 'dynamic_bitset' 'exception' 'flyweight' 'foreach' 'functional' 'fusion' 'geometry' 'graph' 'heap' 'icl' 'integer' 'interprocess' 'intrusive' 'io' 'iostreams' 'iterator' 'lambda2' 'math' 'move' 'mp11' 'mpl' 'multiprecision' 'numeric' 'pending' 'phoenix' 'polygon' 'preprocessor' 'process' 'propery_tree' 'qvm' 'random' 'range' 'scope_exit' 'smart_ptr' 'sort' 'spirit' 'tuple' 'type_traits' 'typeof' 'unordered' 'url' 'utility' 'uuid'.", + "License": "BSL-1.0", + "URL": "https://github.com/eddelbuettel/bh, https://dirk.eddelbuettel.com/code/bh.html", + "BugReports": "https://github.com/eddelbuettel/bh/issues", + "NeedsCompilation": "no", + "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), John W. Emerson [aut], Michael J. Kane [aut] (ORCID: )", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, "KernSmooth": { "Package": "KernSmooth", "Version": "2.23-24", @@ -170,6 +187,81 @@ "Maintainer": "Dirk Eddelbuettel ", "Repository": "CRAN" }, + "RcppArmadillo": { + "Package": "RcppArmadillo", + "Version": "15.2.4-1", + "Source": "Repository", + "Type": "Package", + "Title": "'Rcpp' Integration for the 'Armadillo' Templated Linear Algebra Library", + "Date": "2026-03-17", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Romain\", \"Francois\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Doug\", \"Bates\", role = \"aut\", comment = c(ORCID = \"0000-0001-8316-9503\")), person(\"Binxiang\", \"Ni\", role = \"aut\"), person(\"Conrad\", \"Sanderson\", role = \"aut\", comment = c(ORCID = \"0000-0002-0049-4501\")))", + "Description": "'Armadillo' is a templated C++ linear algebra library aiming towards a good balance between speed and ease of use. It provides high-level syntax and functionality deliberately similar to Matlab. It is useful for algorithm development directly in C++, or quick conversion of research code into production environments. It provides efficient classes for vectors, matrices and cubes where dense and sparse matrices are supported. Integer, floating point and complex numbers are supported. A sophisticated expression evaluator (based on template meta-programming) automatically combines several operations to increase speed and efficiency. Dynamic evaluation automatically chooses optimal code paths based on detected matrix structures. Matrix decompositions are provided through integration with LAPACK, or one of its high performance drop-in replacements (such as 'MKL' or 'OpenBLAS'). It can automatically use 'OpenMP' multi-threading (parallelisation) to speed up computationally expensive operations. . The 'RcppArmadillo' package includes the header files from the 'Armadillo' library; users do not need to install 'Armadillo' itself in order to use 'RcppArmadillo'. Starting from release 15.0.0, the minimum compilation standard is C++14 so 'Armadillo' version 14.6.3 is included as a fallback when an R package forces the C++11 standard. Package authors should set a '#define' to select the 'current' version, or select the 'legacy' version (also chosen as default) if they must. See 'GitHub issue #475' for details. . Since release 7.800.0, 'Armadillo' is licensed under Apache License 2; previous releases were under licensed as MPL 2.0 from version 3.800.0 onwards and LGPL-3 prior to that; 'RcppArmadillo' (the 'Rcpp' bindings/bridge to Armadillo) is licensed under the GNU GPL version 2 or later, as is the rest of 'Rcpp'.", + "License": "GPL (>= 2)", + "LazyLoad": "yes", + "Depends": [ + "R (>= 3.3.0)" + ], + "LinkingTo": [ + "Rcpp" + ], + "Imports": [ + "Rcpp (>= 1.0.12)", + "stats", + "utils", + "methods" + ], + "Suggests": [ + "tinytest", + "Matrix (>= 1.3.0)", + "pkgKitten", + "reticulate", + "slam" + ], + "URL": "https://github.com/RcppCore/RcppArmadillo, https://dirk.eddelbuettel.com/code/rcpp.armadillo.html", + "BugReports": "https://github.com/RcppCore/RcppArmadillo/issues", + "RoxygenNote": "6.0.1", + "NeedsCompilation": "yes", + "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), Romain Francois [aut] (ORCID: ), Doug Bates [aut] (ORCID: ), Binxiang Ni [aut], Conrad Sanderson [aut] (ORCID: )", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, + "RcppEigen": { + "Package": "RcppEigen", + "Version": "0.3.4.0.2", + "Source": "Repository", + "Type": "Package", + "Title": "'Rcpp' Integration for the 'Eigen' Templated Linear Algebra Library", + "Date": "2024-08-23", + "Authors@R": "c(person(\"Doug\", \"Bates\", role = \"aut\", comment = c(ORCID = \"0000-0001-8316-9503\")), person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Romain\", \"Francois\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Yixuan\", \"Qiu\", role = \"aut\", comment = c(ORCID = \"0000-0003-0109-6692\")), person(\"Authors of\", \"Eigen\", role = \"cph\", comment = \"Authorship and copyright in included Eigen library as detailed in inst/COPYRIGHTS\"))", + "Copyright": "See the file COPYRIGHTS for various Eigen copyright details", + "Description": "R and 'Eigen' integration using 'Rcpp'. 'Eigen' is a C++ template library for linear algebra: matrices, vectors, numerical solvers and related algorithms. It supports dense and sparse matrices on integer, floating point and complex numbers, decompositions of such matrices, and solutions of linear systems. Its performance on many algorithms is comparable with some of the best implementations based on 'Lapack' and level-3 'BLAS'. The 'RcppEigen' package includes the header files from the 'Eigen' C++ template library. Thus users do not need to install 'Eigen' itself in order to use 'RcppEigen'. Since version 3.1.1, 'Eigen' is licensed under the Mozilla Public License (version 2); earlier version were licensed under the GNU LGPL version 3 or later. 'RcppEigen' (the 'Rcpp' bindings/bridge to 'Eigen') is licensed under the GNU GPL version 2 or later, as is the rest of 'Rcpp'.", + "License": "GPL (>= 2) | file LICENSE", + "LazyLoad": "yes", + "Depends": [ + "R (>= 3.6.0)" + ], + "LinkingTo": [ + "Rcpp" + ], + "Imports": [ + "Rcpp (>= 0.11.0)", + "stats", + "utils" + ], + "Suggests": [ + "Matrix", + "inline", + "tinytest", + "pkgKitten", + "microbenchmark" + ], + "URL": "https://github.com/RcppCore/RcppEigen, https://dirk.eddelbuettel.com/code/rcpp.eigen.html", + "BugReports": "https://github.com/RcppCore/RcppEigen/issues", + "NeedsCompilation": "yes", + "Author": "Doug Bates [aut] (), Dirk Eddelbuettel [aut, cre] (), Romain Francois [aut] (), Yixuan Qiu [aut] (), Authors of Eigen [cph] (Authorship and copyright in included Eigen library as detailed in inst/COPYRIGHTS)", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, "askpass": { "Package": "askpass", "Version": "1.2.1", @@ -195,6 +287,28 @@ "Maintainer": "Jeroen Ooms ", "Repository": "CRAN" }, + "backports": { + "Package": "backports", + "Version": "1.5.0", + "Source": "Repository", + "Type": "Package", + "Title": "Reimplementations of Functions Introduced Since R-3.0.0", + "Authors@R": "c( person(\"Michel\", \"Lang\", NULL, \"michellang@gmail.com\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0001-9754-0393\")), person(\"Duncan\", \"Murdoch\", NULL, \"murdoch.duncan@gmail.com\", role = c(\"aut\")), person(\"R Core Team\", role = \"aut\"))", + "Maintainer": "Michel Lang ", + "Description": "Functions introduced or changed since R v3.0.0 are re-implemented in this package. The backports are conditionally exported in order to let R resolve the function name to either the implemented backport, or the respective base version, if available. Package developers can make use of new functions or arguments by selectively importing specific backports to support older installations.", + "URL": "https://github.com/r-lib/backports", + "BugReports": "https://github.com/r-lib/backports/issues", + "License": "GPL-2 | GPL-3", + "NeedsCompilation": "yes", + "ByteCompile": "yes", + "Depends": [ + "R (>= 3.0.0)" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "Author": "Michel Lang [cre, aut] (), Duncan Murdoch [aut], R Core Team [aut]", + "Repository": "CRAN" + }, "base64enc": { "Package": "base64enc", "Version": "0.1-3", @@ -212,6 +326,71 @@ "License": "GPL-2 | GPL-3", "URL": "http://www.rforge.net/base64enc", "NeedsCompilation": "yes", + "Repository": "RSPM", + "Encoding": "UTF-8" + }, + "bigmemory": { + "Package": "bigmemory", + "Version": "4.6.4", + "Source": "Repository", + "Title": "Manage Massive Matrices with Shared Memory and Memory-Mapped Files", + "Authors@R": "c( person(given = \"Michael J.\", family = \"Kane\", role = c(\"aut\", \"cre\"), email = \"bigmemoryauthors@gmail.com\", comment = c(ORCID = \"0000-0003-1899-6662\")), person(given = \"John W.\", family = \"Emerson\", role = \"aut\", email = \"jayemerson@gmail.com\"), person(given = \"Peter\", family = \"Haverty\", role = \"aut\", email = \"haverty.peter@gene.com\"), person(given = \"Charles\", family = \"Determan\", role = \"aut\", email = \"cdetermanjr@gmail.com\") )", + "Depends": [ + "R (>= 3.2.0)" + ], + "Imports": [ + "bigmemory.sri", + "methods", + "utils", + "Rcpp", + "uuid (>= 1.0-2)" + ], + "Enhances": [ + "biganalytics", + "bigtabulate" + ], + "LinkingTo": [ + "BH", + "uuid (>= 1.0-2)", + "Rcpp" + ], + "Encoding": "UTF-8", + "Description": "Create, store, access, and manipulate massive matrices. Matrices are allocated to shared memory and may use memory-mapped files. Packages 'biganalytics', 'bigtabulate', 'synchronicity', and 'bigalgebra' provide advanced functionality.", + "License": "LGPL-3 | Apache License 2.0", + "URL": "https://github.com/kaneplusplus/bigmemory", + "BugReports": "https://github.com/kaneplusplus/bigmemory/issues", + "LazyLoad": "yes", + "Suggests": [ + "testthat", + "remotes" + ], + "RoxygenNote": "7.2.3", + "NeedsCompilation": "yes", + "Author": "Michael J. Kane [aut, cre] (), John W. Emerson [aut], Peter Haverty [aut], Charles Determan [aut]", + "Maintainer": "Michael J. Kane ", + "Repository": "CRAN" + }, + "bigmemory.sri": { + "Package": "bigmemory.sri", + "Version": "0.1.8", + "Source": "Repository", + "Type": "Package", + "Title": "A Shared Resource Interface for Bigmemory Project Packages", + "Authors@R": "c( person(given = \"Michael J.\", family = \"Kane\", role = c(\"aut\", \"cre\"), email = \"bigmemoryauthors@gmail.com\", comment = c(ORCID = \"0000-0003-1899-6662\")) )", + "Description": "A shared resource interface for the bigmemory and synchronicity packages.", + "Encoding": "UTF-8", + "Language": "en-US", + "License": "LGPL-3 | Apache License 2.0", + "Enhances": [ + "bigmemory" + ], + "Imports": [ + "methods" + ], + "RoxygenNote": "7.2.3", + "NeedsCompilation": "no", + "Author": "Michael J. Kane [aut, cre] ()", + "Maintainer": "Michael J. Kane ", "Repository": "CRAN" }, "bit": { @@ -807,6 +986,40 @@ "Maintainer": "Jeroen Ooms ", "Repository": "CRAN" }, + "data.table": { + "Package": "data.table", + "Version": "1.18.2.1", + "Source": "Repository", + "Title": "Extension of `data.frame`", + "Depends": [ + "R (>= 3.4.0)" + ], + "Imports": [ + "methods" + ], + "Suggests": [ + "bit64 (>= 4.0.0)", + "bit (>= 4.0.4)", + "R.utils (>= 2.13.0)", + "xts", + "zoo (>= 1.8-1)", + "yaml", + "knitr", + "markdown" + ], + "Description": "Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.", + "License": "MPL-2.0 | file LICENSE", + "URL": "https://r-datatable.com, https://Rdatatable.gitlab.io/data.table, https://github.com/Rdatatable/data.table", + "BugReports": "https://github.com/Rdatatable/data.table/issues", + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "ByteCompile": "TRUE", + "Authors@R": "c( person(\"Tyson\",\"Barrett\", role=c(\"aut\",\"cre\"), email=\"t.barrett88@gmail.com\", comment = c(ORCID=\"0000-0002-2137-1391\")), person(\"Matt\",\"Dowle\", role=\"aut\", email=\"mattjdowle@gmail.com\"), person(\"Arun\",\"Srinivasan\", role=\"aut\", email=\"asrini@pm.me\"), person(\"Jan\",\"Gorecki\", role=\"aut\", email=\"j.gorecki@wit.edu.pl\"), person(\"Michael\",\"Chirico\", role=\"aut\", email=\"michaelchirico4@gmail.com\", comment = c(ORCID=\"0000-0003-0787-087X\")), person(\"Toby\",\"Hocking\", role=\"aut\", email=\"toby.hocking@r-project.org\", comment = c(ORCID=\"0000-0002-3146-0865\")), person(\"Benjamin\",\"Schwendinger\",role=\"aut\", comment = c(ORCID=\"0000-0003-3315-8114\")), person(\"Ivan\", \"Krylov\", role=\"aut\", email=\"ikrylov@disroot.org\", comment = c(ORCID=\"0000-0002-0172-3812\")), person(\"Pasha\",\"Stetsenko\", role=\"ctb\"), person(\"Tom\",\"Short\", role=\"ctb\"), person(\"Steve\",\"Lianoglou\", role=\"ctb\"), person(\"Eduard\",\"Antonyan\", role=\"ctb\"), person(\"Markus\",\"Bonsch\", role=\"ctb\"), person(\"Hugh\",\"Parsonage\", role=\"ctb\"), person(\"Scott\",\"Ritchie\", role=\"ctb\"), person(\"Kun\",\"Ren\", role=\"ctb\"), person(\"Xianying\",\"Tan\", role=\"ctb\"), person(\"Rick\",\"Saporta\", role=\"ctb\"), person(\"Otto\",\"Seiskari\", role=\"ctb\"), person(\"Xianghui\",\"Dong\", role=\"ctb\"), person(\"Michel\",\"Lang\", role=\"ctb\"), person(\"Watal\",\"Iwasaki\", role=\"ctb\"), person(\"Seth\",\"Wenchel\", role=\"ctb\"), person(\"Karl\",\"Broman\", role=\"ctb\"), person(\"Tobias\",\"Schmidt\", role=\"ctb\"), person(\"David\",\"Arenburg\", role=\"ctb\"), person(\"Ethan\",\"Smith\", role=\"ctb\"), person(\"Francois\",\"Cocquemas\", role=\"ctb\"), person(\"Matthieu\",\"Gomez\", role=\"ctb\"), person(\"Philippe\",\"Chataignon\", role=\"ctb\"), person(\"Nello\",\"Blaser\", role=\"ctb\"), person(\"Dmitry\",\"Selivanov\", role=\"ctb\"), person(\"Andrey\",\"Riabushenko\", role=\"ctb\"), person(\"Cheng\",\"Lee\", role=\"ctb\"), person(\"Declan\",\"Groves\", role=\"ctb\"), person(\"Daniel\",\"Possenriede\", role=\"ctb\"), person(\"Felipe\",\"Parages\", role=\"ctb\"), person(\"Denes\",\"Toth\", role=\"ctb\"), person(\"Mus\",\"Yaramaz-David\", role=\"ctb\"), person(\"Ayappan\",\"Perumal\", role=\"ctb\"), person(\"James\",\"Sams\", role=\"ctb\"), person(\"Martin\",\"Morgan\", role=\"ctb\"), person(\"Michael\",\"Quinn\", role=\"ctb\"), person(given=\"@javrucebo\", role=\"ctb\", comment=\"GitHub user\"), person(\"Marc\",\"Halperin\", role=\"ctb\"), person(\"Roy\",\"Storey\", role=\"ctb\"), person(\"Manish\",\"Saraswat\", role=\"ctb\"), person(\"Morgan\",\"Jacob\", role=\"ctb\"), person(\"Michael\",\"Schubmehl\", role=\"ctb\"), person(\"Davis\",\"Vaughan\", role=\"ctb\"), person(\"Leonardo\",\"Silvestri\", role=\"ctb\"), person(\"Jim\",\"Hester\", role=\"ctb\"), person(\"Anthony\",\"Damico\", role=\"ctb\"), person(\"Sebastian\",\"Freundt\", role=\"ctb\"), person(\"David\",\"Simons\", role=\"ctb\"), person(\"Elliott\",\"Sales de Andrade\", role=\"ctb\"), person(\"Cole\",\"Miller\", role=\"ctb\"), person(\"Jens Peder\",\"Meldgaard\", role=\"ctb\"), person(\"Vaclav\",\"Tlapak\", role=\"ctb\"), person(\"Kevin\",\"Ushey\", role=\"ctb\"), person(\"Dirk\",\"Eddelbuettel\", role=\"ctb\"), person(\"Tony\",\"Fischetti\", role=\"ctb\"), person(\"Ofek\",\"Shilon\", role=\"ctb\"), person(\"Vadim\",\"Khotilovich\", role=\"ctb\"), person(\"Hadley\",\"Wickham\", role=\"ctb\"), person(\"Bennet\",\"Becker\", role=\"ctb\"), person(\"Kyle\",\"Haynes\", role=\"ctb\"), person(\"Boniface Christian\",\"Kamgang\", role=\"ctb\"), person(\"Olivier\",\"Delmarcell\", role=\"ctb\"), person(\"Josh\",\"O'Brien\", role=\"ctb\"), person(\"Dereck\",\"de Mezquita\", role=\"ctb\"), person(\"Michael\",\"Czekanski\", role=\"ctb\"), person(\"Dmitry\", \"Shemetov\", role=\"ctb\"), person(\"Nitish\", \"Jha\", role=\"ctb\"), person(\"Joshua\", \"Wu\", role=\"ctb\"), person(\"Iago\", \"Giné-Vázquez\", role=\"ctb\"), person(\"Anirban\", \"Chetia\", role=\"ctb\"), person(\"Doris\", \"Amoakohene\", role=\"ctb\"), person(\"Angel\", \"Feliz\", role=\"ctb\"), person(\"Michael\",\"Young\", role=\"ctb\"), person(\"Mark\", \"Seeto\", role=\"ctb\"), person(\"Philippe\", \"Grosjean\", role=\"ctb\"), person(\"Vincent\", \"Runge\", role=\"ctb\"), person(\"Christian\", \"Wia\", role=\"ctb\"), person(\"Elise\", \"Maigné\", role=\"ctb\"), person(\"Vincent\", \"Rocher\", role=\"ctb\"), person(\"Vijay\", \"Lulla\", role=\"ctb\"), person(\"Aljaž\", \"Sluga\", role=\"ctb\"), person(\"Bill\", \"Evans\", role=\"ctb\"), person(\"Reino\", \"Bruner\", role=\"ctb\"), person(given=\"@badasahog\", role=\"ctb\", comment=\"GitHub user\"), person(\"Vinit\", \"Thakur\", role=\"ctb\"), person(\"Mukul\", \"Kumar\", role=\"ctb\"), person(\"Ildikó\", \"Czeller\", role=\"ctb\"), person(\"Manmita\", \"Das\", role=\"ctb\") )", + "NeedsCompilation": "yes", + "Author": "Tyson Barrett [aut, cre] (ORCID: ), Matt Dowle [aut], Arun Srinivasan [aut], Jan Gorecki [aut], Michael Chirico [aut] (ORCID: ), Toby Hocking [aut] (ORCID: ), Benjamin Schwendinger [aut] (ORCID: ), Ivan Krylov [aut] (ORCID: ), Pasha Stetsenko [ctb], Tom Short [ctb], Steve Lianoglou [ctb], Eduard Antonyan [ctb], Markus Bonsch [ctb], Hugh Parsonage [ctb], Scott Ritchie [ctb], Kun Ren [ctb], Xianying Tan [ctb], Rick Saporta [ctb], Otto Seiskari [ctb], Xianghui Dong [ctb], Michel Lang [ctb], Watal Iwasaki [ctb], Seth Wenchel [ctb], Karl Broman [ctb], Tobias Schmidt [ctb], David Arenburg [ctb], Ethan Smith [ctb], Francois Cocquemas [ctb], Matthieu Gomez [ctb], Philippe Chataignon [ctb], Nello Blaser [ctb], Dmitry Selivanov [ctb], Andrey Riabushenko [ctb], Cheng Lee [ctb], Declan Groves [ctb], Daniel Possenriede [ctb], Felipe Parages [ctb], Denes Toth [ctb], Mus Yaramaz-David [ctb], Ayappan Perumal [ctb], James Sams [ctb], Martin Morgan [ctb], Michael Quinn [ctb], @javrucebo [ctb] (GitHub user), Marc Halperin [ctb], Roy Storey [ctb], Manish Saraswat [ctb], Morgan Jacob [ctb], Michael Schubmehl [ctb], Davis Vaughan [ctb], Leonardo Silvestri [ctb], Jim Hester [ctb], Anthony Damico [ctb], Sebastian Freundt [ctb], David Simons [ctb], Elliott Sales de Andrade [ctb], Cole Miller [ctb], Jens Peder Meldgaard [ctb], Vaclav Tlapak [ctb], Kevin Ushey [ctb], Dirk Eddelbuettel [ctb], Tony Fischetti [ctb], Ofek Shilon [ctb], Vadim Khotilovich [ctb], Hadley Wickham [ctb], Bennet Becker [ctb], Kyle Haynes [ctb], Boniface Christian Kamgang [ctb], Olivier Delmarcell [ctb], Josh O'Brien [ctb], Dereck de Mezquita [ctb], Michael Czekanski [ctb], Dmitry Shemetov [ctb], Nitish Jha [ctb], Joshua Wu [ctb], Iago Giné-Vázquez [ctb], Anirban Chetia [ctb], Doris Amoakohene [ctb], Angel Feliz [ctb], Michael Young [ctb], Mark Seeto [ctb], Philippe Grosjean [ctb], Vincent Runge [ctb], Christian Wia [ctb], Elise Maigné [ctb], Vincent Rocher [ctb], Vijay Lulla [ctb], Aljaž Sluga [ctb], Bill Evans [ctb], Reino Bruner [ctb], @badasahog [ctb] (GitHub user), Vinit Thakur [ctb], Mukul Kumar [ctb], Ildikó Czeller [ctb], Manmita Das [ctb]", + "Maintainer": "Tyson Barrett ", + "Repository": "CRAN" + }, "desc": { "Package": "desc", "Version": "1.4.3", @@ -1180,6 +1393,42 @@ "Maintainer": "Brodie Gaslam ", "Repository": "CRAN" }, + "fastglm": { + "Package": "fastglm", + "Version": "0.0.4", + "Source": "Repository", + "Type": "Package", + "Title": "Fast and Stable Fitting of Generalized Linear Models using 'RcppEigen'", + "Authors@R": "c( person(\"Jared\", \"Huling\", , \"jaredhuling@gmail.com\", c(\"aut\", \"cre\")), person(\"Douglas\", \"Bates\", , , c(\"cph\")), person(\"Dirk\", \"Eddelbuettel\", , , c(\"cph\")), person(\"Romain\", \"Francois\", , , c(\"cph\")), person(\"Yixuan\", \"Qiu\", , , c(\"cph\")) )", + "Maintainer": "Jared Huling ", + "Description": "Fits generalized linear models efficiently using 'RcppEigen'. The iteratively reweighted least squares implementation utilizes the step-halving approach of Marschner (2011) to help safeguard against convergence issues.", + "BugReports": "https://github.com/jaredhuling/fastglm/issues", + "License": "GPL (>= 2)", + "Encoding": "UTF-8", + "Imports": [ + "Rcpp (>= 0.12.13)", + "methods" + ], + "Depends": [ + "bigmemory" + ], + "LinkingTo": [ + "Rcpp", + "RcppEigen", + "BH", + "bigmemory" + ], + "RoxygenNote": "7.3.3", + "Suggests": [ + "knitr", + "rmarkdown", + "glm2" + ], + "VignetteBuilder": "knitr", + "NeedsCompilation": "yes", + "Author": "Jared Huling [aut, cre], Douglas Bates [cph], Dirk Eddelbuettel [cph], Romain Francois [cph], Yixuan Qiu [cph]", + "Repository": "CRAN" + }, "fastmap": { "Package": "fastmap", "Version": "1.2.0", @@ -2168,6 +2417,42 @@ "Maintainer": "Joe Cheng ", "Repository": "CRAN" }, + "mvnfast": { + "Package": "mvnfast", + "Version": "0.2.8", + "Source": "Repository", + "Type": "Package", + "Title": "Fast Multivariate Normal and Student's t Methods", + "Date": "2023-02-20", + "Authors@R": "c(person(\"Matteo\", \"Fasiolo\", email = \"matteo.fasiolo@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Thijs\", \" van den Berg\", role = c(\"ctb\")))", + "Maintainer": "Matteo Fasiolo ", + "Description": "Provides computationally efficient tools related to the multivariate normal and Student's t distributions. The main functionalities are: simulating multivariate random vectors, evaluating multivariate normal or Student's t densities and Mahalanobis distances. These tools are very efficient thanks to the use of C++ code and of the OpenMP API.", + "License": "GPL (>= 2.0)", + "URL": "https://github.com/mfasiolo/mvnfast/", + "Imports": [ + "Rcpp" + ], + "Suggests": [ + "knitr", + "rmarkdown", + "testthat", + "mvtnorm", + "microbenchmark", + "MASS", + "plyr", + "RhpcBLASctl" + ], + "LinkingTo": [ + "Rcpp", + "RcppArmadillo", + "BH" + ], + "VignetteBuilder": "knitr", + "RoxygenNote": "7.2.2", + "NeedsCompilation": "yes", + "Author": "Matteo Fasiolo [aut, cre], Thijs van den Berg [ctb]", + "Repository": "CRAN" + }, "nlme": { "Package": "nlme", "Version": "3.1-166", @@ -2296,6 +2581,34 @@ "Maintainer": "Gábor Csárdi ", "Repository": "CRAN" }, + "pbv": { + "Package": "pbv", + "Version": "0.5-47", + "Source": "Repository", + "Type": "Package", + "Title": "Probabilities for Bivariate Normal Distribution", + "Date": "2023-11-30 15:01:12.000296", + "Author": "Alexander Robitzsch [aut,cre] ()", + "Maintainer": "Alexander Robitzsch ", + "Description": "Computes probabilities of the bivariate normal distribution in a vectorized R function (Drezner & Wesolowsky, 1990, ).", + "Depends": [ + "R (>= 3.1)" + ], + "Imports": [ + "Rcpp" + ], + "Enhances": [ + "pbivnorm" + ], + "LinkingTo": [ + "Rcpp", + "RcppArmadillo" + ], + "URL": "https://github.com/alexanderrobitzsch/pbv, https://sites.google.com/view/alexander-robitzsch/software", + "License": "GPL (>= 2)", + "NeedsCompilation": "yes", + "Repository": "CRAN" + }, "pillar": { "Package": "pillar", "Version": "1.11.1", @@ -3569,6 +3882,72 @@ "Maintainer": "Winston Chang ", "Repository": "CRAN" }, + "simstudy": { + "Package": "simstudy", + "Version": "0.9.2", + "Source": "Repository", + "Type": "Package", + "Title": "Simulation of Study Data", + "Date": "2026-02-06", + "Authors@R": "c(person(given = \"Keith\", family = \"Goldfeld\", role = c(\"aut\", \"cre\"), email = \"keith.goldfeld@nyulangone.org\", comment = c(ORCID = \"0000-0002-0292-8780\")), person(given = \"Jacob\", family = \"Wujciak-Jens\", role = \"aut\", email = \"jacob@wujciak.de\", comment = c(ORCID = \"0000-0002-7281-3989\")))", + "Description": "Simulates data sets in order to explore modeling techniques or better understand data generating processes. The user specifies a set of relationships between covariates, and generates data based on these specifications. The final data sets can represent data from randomized control trials, repeated measure (longitudinal) designs, and cluster randomized trials. Missingness can be generated using various mechanisms (MCAR, MAR, NMAR).", + "License": "GPL-3", + "URL": "https://github.com/kgoldfeld/simstudy, https://kgoldfeld.github.io/simstudy/, https://kgoldfeld.github.io/simstudy/dev/", + "BugReports": "https://github.com/kgoldfeld/simstudy/issues", + "Depends": [ + "R (>= 4.1.0)" + ], + "Imports": [ + "data.table", + "glue", + "methods", + "mvnfast", + "Rcpp", + "backports", + "fastglm", + "pbv (>= 0.5.47)" + ], + "Suggests": [ + "covr", + "dplyr", + "formatR", + "gee", + "ggplot2", + "grid", + "gridExtra", + "hedgehog", + "knitr", + "magrittr", + "Matrix", + "mgcv", + "ordinal", + "pracma", + "rmarkdown", + "scales", + "splines", + "survival", + "testthat", + "gtsummary", + "broom.helpers", + "survminer", + "katex", + "dirmult", + "rms", + "lmerTest" + ], + "LinkingTo": [ + "Rcpp", + "pbv (>= 0.5.47)", + "fastglm" + ], + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Keith Goldfeld [aut, cre] (ORCID: ), Jacob Wujciak-Jens [aut] (ORCID: )", + "Maintainer": "Keith Goldfeld ", + "Repository": "CRAN" + }, "sourcetools": { "Package": "sourcetools", "Version": "0.1.7-1", @@ -4209,6 +4588,24 @@ "Maintainer": "Kirill Müller ", "Repository": "CRAN" }, + "uuid": { + "Package": "uuid", + "Version": "1.2-2", + "Source": "Repository", + "Title": "Tools for Generating and Handling of UUIDs", + "Author": "Simon Urbanek [aut, cre, cph] (https://urbanek.org, ORCID: ), Theodore Ts'o [aut, cph] (libuuid)", + "Maintainer": "Simon Urbanek ", + "Authors@R": "c(person(\"Simon\", \"Urbanek\", role=c(\"aut\",\"cre\",\"cph\"), email=\"Simon.Urbanek@r-project.org\", comment=c(\"https://urbanek.org\", ORCID=\"0000-0003-2297-1732\")), person(\"Theodore\",\"Ts'o\", email=\"tytso@thunk.org\", role=c(\"aut\",\"cph\"), comment=\"libuuid\"))", + "Depends": [ + "R (>= 2.9.0)" + ], + "Description": "Tools for generating and handling of UUIDs (Universally Unique Identifiers).", + "License": "MIT + file LICENSE", + "URL": "https://www.rforge.net/uuid", + "BugReports": "https://github.com/s-u/uuid/issues", + "NeedsCompilation": "yes", + "Repository": "CRAN" + }, "vctrs": { "Package": "vctrs", "Version": "0.6.5", @@ -4562,9 +4959,10 @@ "R (>= 2.10.0)" ], "License": "GPL (>= 2)", - "Repository": "CRAN", + "Repository": "RSPM", "NeedsCompilation": "no", - "Author": "David B. Dahl [aut], David Scott [aut, cre], Charles Roosen [aut], Arni Magnusson [aut], Jonathan Swinton [aut], Ajay Shah [ctb], Arne Henningsen [ctb], Benno Puetz [ctb], Bernhard Pfaff [ctb], Claudio Agostinelli [ctb], Claudius Loehnert [ctb], David Mitchell [ctb], David Whiting [ctb], Fernando da Rosa [ctb], Guido Gay [ctb], Guido Schulz [ctb], Ian Fellows [ctb], Jeff Laake [ctb], John Walker [ctb], Jun Yan [ctb], Liviu Andronic [ctb], Markus Loecher [ctb], Martin Gubri [ctb], Matthieu Stigler [ctb], Robert Castelo [ctb], Seth Falcon [ctb], Stefan Edwards [ctb], Sven Garbade [ctb], Uwe Ligges [ctb]" + "Author": "David B. Dahl [aut], David Scott [aut, cre], Charles Roosen [aut], Arni Magnusson [aut], Jonathan Swinton [aut], Ajay Shah [ctb], Arne Henningsen [ctb], Benno Puetz [ctb], Bernhard Pfaff [ctb], Claudio Agostinelli [ctb], Claudius Loehnert [ctb], David Mitchell [ctb], David Whiting [ctb], Fernando da Rosa [ctb], Guido Gay [ctb], Guido Schulz [ctb], Ian Fellows [ctb], Jeff Laake [ctb], John Walker [ctb], Jun Yan [ctb], Liviu Andronic [ctb], Markus Loecher [ctb], Martin Gubri [ctb], Matthieu Stigler [ctb], Robert Castelo [ctb], Seth Falcon [ctb], Stefan Edwards [ctb], Sven Garbade [ctb], Uwe Ligges [ctb]", + "Encoding": "UTF-8" }, "yaml": { "Package": "yaml", @@ -4583,7 +4981,8 @@ "URL": "https://github.com/vubiostat/r-yaml/", "BugReports": "https://github.com/vubiostat/r-yaml/issues", "NeedsCompilation": "yes", - "Repository": "CRAN" + "Repository": "RSPM", + "Encoding": "UTF-8" }, "zip": { "Package": "zip", diff --git a/tests/testthat/test-edge-case-contract.R b/tests/testthat/test-edge-case-contract.R index d65339e..555dca4 100644 --- a/tests/testthat/test-edge-case-contract.R +++ b/tests/testthat/test-edge-case-contract.R @@ -25,6 +25,22 @@ test_that("native backend returns typed zero-row output for n = 0", { expect_type(result$age, "double") }) +test_that("simstudy backend matches the native zero-row contract for n = 0", { + # Issue #50: simstudy::genData(0, def) builds its id table as + # data.table(x = 1:n), and 1:0 is c(1, 0) -- two rows -- so evaluation + # stops with a length mismatch. Both simstudy-eligible types are covered. + skip_if_not_installed("simstudy") + spec <- mock_spec( + mock_spec_categorical("smoking", levels = c("never", "former", "current")), + mock_spec_continuous("age", range = c(18, 80)) + ) + result <- generate_mock_data_simstudy(spec, n = 0, seed = 1) + expect_identical(result, generate_mock_data_native(spec, n = 0, seed = 1)) + expect_identical(nrow(result), 0L) + expect_named(result, c("smoking", "age")) + expect_type(result$age, "double") +}) + test_that("native backend returns n rows and zero columns for an empty spec", { expect_identical(dim(generate_mock_data_native(mock_spec(), n = 10)), c(10L, 0L)) expect_identical(dim(generate_mock_data_native(mock_spec(), n = 0)), c(0L, 0L)) diff --git a/tests/testthat/test-formula-vars.R b/tests/testthat/test-formula-vars.R new file mode 100644 index 0000000..bd87bf1 --- /dev/null +++ b/tests/testthat/test-formula-vars.R @@ -0,0 +1,449 @@ +# tests/testthat/test-formula-vars.R +# Formula-derived variables (#39, Phase A). ADR: development/adr/v05-formula-evaluator.md + +test_that("mock_formula constructs a validated formula variable", { + spec <- mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_continuous("weight", range = c(45, 150)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ) + expect_true(validate_mock_spec(spec)$valid) + expect_identical(spec$variables$bmi$type, "formula") + expect_identical(spec$variables$bmi$formula, "weight / (height^2)") + expect_setequal(spec$variables$bmi$depends_on, c("weight", "height")) +}) + +test_that("a formula referencing an unknown variable fails validation", { + expect_error( + mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ), + "references unknown variable" + ) +}) + +test_that("a disallowed function symbol fails validation before generation", { + expect_error( + mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "system('echo pwned') + x") + ), + "not permitted in mockFormula" + ) +}) + +test_that("a dependency cycle among formula variables errors clearly", { + expect_error( + mock_spec( + mock_spec_formula("a", formula = "b + 1"), + mock_spec_formula("b", formula = "a + 1") + ), + "dependency cycle" + ) +}) + +test_that("an unparseable formula errors at validation", { + expect_error( + mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2") + ), + "could not be parsed" + ) +}) + +test_that("mock_spec_formula() defers parse errors instead of throwing eagerly", { + # The constructor itself must not throw on bad syntax; depends_on falls + # back to character(0) and the parse error is reported by validation. + variable <- mock_spec_formula("bad", formula = "x +* 2") + expect_identical(variable$depends_on, character(0)) +}) + +test_that("validate = FALSE constructs a spec with a bad-syntax formula", { + expect_no_error( + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2"), + validate = FALSE + ) + ) + + result <- validate_mock_spec(spec, strict = FALSE) + expect_false(result$valid) + expect_true(any(grepl("could not be parsed", result$errors))) +}) + +test_that("validate_mock_spec accumulates errors for multiple bad-syntax formulas", { + spec <- mock_spec( + mock_spec_formula("bad1", formula = "x +* 2"), + mock_spec_formula("bad2", formula = "y +* 3"), + validate = FALSE + ) + + result <- validate_mock_spec(spec, strict = FALSE) + expect_false(result$valid) + expect_true(any(grepl("bad1", result$errors) & grepl("could not be parsed", result$errors))) + expect_true(any(grepl("bad2", result$errors) & grepl("could not be parsed", result$errors))) +}) + +test_that("an unparseable formula does not mask referent errors on other formulas", { + # Folded-in review finding from Task 1: spec-level checks re-parsed every + # formula and failed fast, so one unparseable formula both (a) produced a + # duplicate "could not be parsed" error alongside the per-variable one and + # (b) masked referent/cycle findings for the OTHER, parseable formulas. + # Fix: .validate_formula_referents()/.order_formula_variables() must skip + # variables whose formula fails to parse. + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2"), + mock_spec_formula("orphan", formula = "unknown_var + 1"), + validate = FALSE + ) + + result <- validate_mock_spec(spec, strict = FALSE) + expect_false(result$valid) + + parse_errors <- grepl("bad", result$errors) & grepl("could not be parsed", result$errors) + expect_identical(sum(parse_errors), 1L) # no duplicate parse entry + + referent_errors <- grepl("orphan", result$errors) & grepl("references unknown variable", result$errors) + expect_true(any(referent_errors)) +}) + +test_that("evaluate_mock_formulas computes algebraic derivations correctly", { + spec <- mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_continuous("weight", range = c(45, 150)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ) + baseline <- generate_mock_data_native(spec, n = 50, seed = 11) + expect_false("bmi" %in% names(baseline)) # native skips formula vars + result <- evaluate_mock_formulas(baseline, spec, seed = 11) + expect_identical(result$bmi, baseline$weight / (baseline$height^2)) + expect_identical(result$height, baseline$height) # inputs untouched +}) + +test_that("formula-of-a-formula evaluates in dependency order", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 10)), + mock_spec_formula("obese_flag", formula = "ifelse(y > 15, 1, 0)", rtype = "integer"), + mock_spec_formula("y", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 20, seed = 5) + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(result$y, baseline$x * 2) + expect_identical(result$obese_flag, as.integer(ifelse(baseline$x * 2 > 15, 1L, 0L))) +}) + +test_that("evaluation cannot reach the caller's environment", { + leaky <- 999 + # 'leaky' is not a spec variable -> referent validation rejects it at + # construction time (eager validation, confirmed in Task 1), before the + # sandbox is ever exercised. The sandbox property itself is pinned by the + # eval-environment test below. + expect_error( + mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x + leaky") + ), + "references unknown variable" + ) +}) + +test_that("eval environment exposes only columns and allow-listed functions", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "sqrt(x)") + ) + baseline <- generate_mock_data_native(spec, n = 5, seed = 3) + env_probe <- evaluate_mock_formulas(baseline, spec) + expect_identical(env_probe$z, sqrt(baseline$x)) +}) + +test_that("evaluate_mock_formulas is a no-op without formula variables", { + spec <- mock_continuous("x", range = c(0, 1)) + baseline <- generate_mock_data_native(spec, n = 5, seed = 3) + expect_identical(evaluate_mock_formulas(baseline, spec), baseline) +}) + +test_that("evaluate_mock_formulas handles n = 0", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 0, seed = 3) + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(nrow(result), 0L) + expect_true("z" %in% names(result)) +}) + +test_that("formula stage leaves the caller's RNG untouched and is deterministic", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 10, seed = 7) + set.seed(123); before <- .Random.seed + a <- evaluate_mock_formulas(baseline, spec, seed = 7) + expect_identical(.Random.seed, before) + expect_identical(a, evaluate_mock_formulas(baseline, spec, seed = 7)) +}) + +test_that("rtype coercion applies to formula outputs", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 9)), + mock_spec_formula("band", formula = "ifelse(x > 5, 'high', 'low')", rtype = "factor") + ) + baseline <- generate_mock_data_native(spec, n = 20, seed = 9) + result <- evaluate_mock_formulas(baseline, spec) + expect_s3_class(result$band, "factor") +}) + +test_that("evaluate_mock_formulas errors rather than silently dropping an unparseable formula", { + # Task 2 review finding 1: .order_formula_variables() filters unparseable + # formulas out of `ordered` (correct for validate_mock_spec's accumulation), + # but the evaluator conflated "no formula variables" with "no *parseable* + # formula variables" and returned `data` unchanged with no error. Baseline + # is built directly (not via generate_mock_data_native()), because that + # generator's own entry-point strict validation would already reject this + # spec before we can reach evaluate_mock_formulas(). + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2"), + validate = FALSE + ) + baseline <- data.frame(x = c(0.1, 0.2, 0.3)) + + expect_error( + evaluate_mock_formulas(baseline, spec), + "could not be parsed" + ) +}) + +test_that("evaluate_mock_formulas errors (not partial output) with a mixed bad + good formula spec", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad", formula = "x +* 2"), + mock_spec_formula("good", formula = "x * 2"), + validate = FALSE + ) + baseline <- data.frame(x = c(0.1, 0.2, 0.3)) + + expect_error( + evaluate_mock_formulas(baseline, spec), + "could not be parsed" + ) +}) + +test_that("evaluate_mock_formulas errors clearly on a length-mismatched formula result", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("bad_len", formula = "c(1, 2, 3)") + ) + baseline <- generate_mock_data_native(spec, n = 10, seed = 1) + + expect_error( + evaluate_mock_formulas(baseline, spec), + "returned length 3, expected 10" + ) +}) + +test_that("evaluate_mock_formulas works for nrow == 1 with a scalar formula result (no rep needed)", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(0, 1)), + mock_spec_formula("z", formula = "x * 2") + ) + baseline <- generate_mock_data_native(spec, n = 1, seed = 1) + + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(nrow(result), 1L) + expect_identical(result$z, baseline$x * 2) +}) + +test_that("evaluate_mock_formulas handles a formula-only spec via a constant, broadcasting formula", { + # A formula-only spec has no non-formula columns to reference, so its + # baseline is an n-row, zero-column data frame (see + # generate_mock_data_native()'s formula skip). A constant formula still + # broadcasts correctly over that 0-column frame (verified manually before + # pinning: length(values) == 1 && nrow(data) != 1 triggers the rep() path). + spec <- mock_spec(mock_spec_formula("k", formula = "1 + 1")) + baseline <- generate_mock_data_native(spec, n = 7, seed = 1) + expect_identical(dim(baseline), c(7L, 0L)) + + result <- evaluate_mock_formulas(baseline, spec) + expect_identical(result$k, rep(2, 7)) +}) + +# ------------------------------------------------------------------------ +# Task 3: adapter + orchestrator wiring, diagnostics, end-to-end (#39) +# ------------------------------------------------------------------------ + +test_that("mockFormula metadata generates a derived column end-to-end", { + variables <- data.frame( + variable = c("height", "weight", "bmi"), + variableType = c("Continuous", "Continuous", "Continuous"), + rType = c("double", "double", "double"), + role = c("enabled", "enabled", "enabled"), + stringsAsFactors = FALSE + ) + variable_details <- data.frame( + variable = c("height", "weight", "bmi"), + recStart = c("[1.4, 2.1]", "[45, 150]", "DerivedVar::[height, weight]"), + recEnd = c("copy", "copy", "Func::bmi_fun"), + proportion = c(1, 1, 1), + mockFormula = c("", "", "weight / (height^2)"), + stringsAsFactors = FALSE + ) + + result <- suppressMessages(create_mock_data( + databaseStart = "study", + variables = variables, + variable_details = variable_details, + n = 30, + seed = 4 + )) + + expect_true("bmi" %in% names(result)) + diag <- attr(result, "mockdata_diagnostics") + expect_false(is.null(diag)) + expect_true(isTRUE(diag$variables$bmi$derived)) + expect_setequal(diag$variables$bmi$depends_on, c("height", "weight")) +}) + +test_that("DerivedVar rows without mockFormula stay excluded (D6, backward compat)", { + variables <- data.frame( + variable = c("height", "weight", "bmi"), + variableType = c("Continuous", "Continuous", "Continuous"), + rType = c("double", "double", "double"), + role = c("enabled", "enabled", "enabled"), + stringsAsFactors = FALSE + ) + variable_details <- data.frame( + variable = c("height", "weight", "bmi"), + recStart = c("[1.4, 2.1]", "[45, 150]", "DerivedVar::[height, weight]"), + recEnd = c("copy", "copy", "Func::bmi_fun"), + proportion = c(1, 1, 1), + stringsAsFactors = FALSE + ) + + result <- suppressMessages(create_mock_data( + databaseStart = "study", + variables = variables, + variable_details = variable_details, + n = 10, + seed = 4 + )) + + expect_false("bmi" %in% names(result)) +}) + +test_that("D6 carve-out is scoped per databaseStart, not cross-cycle (#39 review finding)", { + # bmi has a mockFormula in cycle1 but not in cycle2. Requesting cycle2 must + # exclude bmi exactly as it would have before the mockFormula feature + # existed -- construction is databaseStart-scoped, so the exclude_derived + # carve-out must be too, or a cycle1-only formula leaks into cycle2's + # decision and construction crashes trying to build bmi as a continuous + # variable from a DerivedVar:: recStart. + variables <- data.frame( + variable = c("height", "weight", "bmi"), + variableType = c("Continuous", "Continuous", "Continuous"), + rType = c("double", "double", "double"), + role = c("enabled", "enabled", "enabled"), + databaseStart = c("", "", ""), + stringsAsFactors = FALSE + ) + variable_details <- data.frame( + variable = c("height", "weight", "bmi", "bmi"), + recStart = c( + "[1.4, 2.1]", "[45, 150]", + "DerivedVar::[height, weight]", "DerivedVar::[height, weight]" + ), + recEnd = c("copy", "copy", "Func::bmi_fun", "Func::bmi_fun"), + proportion = c(1, 1, 1, 1), + mockFormula = c("", "", "weight / (height^2)", ""), + databaseStart = c("", "", "cycle1", "cycle2"), + stringsAsFactors = FALSE + ) + + expect_no_error( + cycle2_spec <- mock_spec_from_recodeflow( + variables, variable_details, + databaseStart = "cycle2", role = "enabled" + ) + ) + expect_false("bmi" %in% names(cycle2_spec$variables)) + + cycle1_spec <- mock_spec_from_recodeflow( + variables, variable_details, + databaseStart = "cycle1", role = "enabled" + ) + expect_true("bmi" %in% names(cycle1_spec$variables)) + expect_identical(cycle1_spec$variables$bmi$type, "formula") +}) + +test_that("formula variables receive their own missing codes in postprocess", { + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 9)), + mock_spec_formula("z", formula = "x * 2", + missing_codes = -99, missing_proportions = 0.3) + ) + baseline <- generate_mock_data_native(spec, n = 200, seed = 8) + staged <- evaluate_mock_formulas(baseline, spec, seed = 8) + result <- postprocess_mock_data(staged, spec, seed = 8) + expect_true(any(result$z == -99)) + expect_true(all(result$z[result$z != -99] == staged$z[result$z != -99])) +}) + +test_that("existing formula-free seeded output is unchanged by the formula stage", { + # The #48 pinned-reference tests already enforce this mechanically; this + # test additionally pins the orchestrator route: a formula-free spec must + # not pass through evaluate_mock_formulas' seeded wrapper at all. + spec <- mock_continuous("x", range = c(0, 1)) + baseline <- generate_mock_data_native(spec, n = 3, seed = 20260706) + expect_identical(evaluate_mock_formulas(baseline, spec, seed = 20260706), baseline) +}) + +test_that("warn when a formula produces a factor but rtype is numeric", { + # Whole-branch review Minor #1: factor coercion to numeric silently returns + # level codes. Warn to make this foreseeable footgun obvious. + spec <- mock_spec( + mock_spec_continuous("x", range = c(1, 10)), + mock_spec_formula("band", formula = "cut(x, c(0,3,7,10))", rtype = "double") + ) + baseline <- generate_mock_data_native(spec, n = 5, seed = 42) + + expect_warning( + result <- evaluate_mock_formulas(baseline, spec), + "formula produced a factor but rType is 'double'" + ) + # Verify coercion still happens (to level codes) + expect_true(is.numeric(result$band)) +}) + +test_that("warn when variable details have conflicting mockFormula values", { + # Whole-branch review Minor #2: conflicting mockFormula values silently use + # the first. Warn when distinct values are detected. + variables <- data.frame( + variable = c("x", "z"), + variableType = c("Continuous", "Continuous"), + rType = c("double", "double"), + role = c("enabled", "enabled"), + stringsAsFactors = FALSE + ) + variable_details <- data.frame( + variable = c("x", "z", "z"), + recStart = c("[1, 10]", "DerivedVar::[x]", "DerivedVar::[x]"), + recEnd = c("copy", "Func::foo", "Func::foo"), + proportion = c(1, 1, 1), + mockFormula = c("", "x * 2", "x * 3"), # conflicting: "x * 2" vs "x * 3" + stringsAsFactors = FALSE + ) + + expect_warning( + spec <- mock_spec_from_recodeflow(variables, variable_details), + "conflicting mockFormula values.*using the first: 'x \\* 2'" + ) + # Verify the first value was used + expect_identical(spec$variables$z$formula, "x * 2") +}) diff --git a/tests/testthat/test-range-missing-codes.R b/tests/testthat/test-range-missing-codes.R new file mode 100644 index 0000000..5bdb7f3 --- /dev/null +++ b/tests/testthat/test-range-missing-codes.R @@ -0,0 +1,121 @@ +# tests/testthat/test-range-missing-codes.R +# Range-notation missing codes such as "[997,999]" on the v0.4 path (#58). +# recodeflow writes grouped missing codes (don't know / refusal / not stated) +# as integer ranges; the adapter must expand them into individual codes. + +range_continuous_metadata <- function(missing_start = "[997,999]") { + list( + variables = data.frame( + variable = "age", + variableType = "Continuous", + rType = "integer", + role = "enabled", + stringsAsFactors = FALSE + ), + variable_details = data.frame( + variable = "age", + recStart = c("[18,85]", "996", missing_start), + recEnd = c("copy", "NA::a", "NA::b"), + proportion = c(NA, 0.05, 0.09), + stringsAsFactors = FALSE + ) + ) +} + +range_categorical_metadata <- function() { + list( + variables = data.frame( + variable = "smoke", + variableType = "Categorical", + rType = "character", + role = "enabled", + stringsAsFactors = FALSE + ), + variable_details = data.frame( + variable = "smoke", + recStart = c("1", "2", "6", "[7, 9]"), + recEnd = c("1", "2", "NA::a", "NA::b"), + proportion = c(0.5, 0.3, 0.05, 0.15), + stringsAsFactors = FALSE + ) + ) +} + +test_that("the adapter expands an integer range missing code on a continuous variable", { + md <- range_continuous_metadata() + spec <- mock_spec_from_recodeflow(md$variables, md$variable_details) + age <- spec$variables$age + expect_identical(age$missing_codes, c("996", "997", "998", "999")) + expect_equal(age$missing_proportions, c(0.05, 0.03, 0.03, 0.03)) +}) + +test_that("the adapter expands an integer range missing code on a categorical variable", { + md <- range_categorical_metadata() + spec <- suppressWarnings(mock_spec_from_recodeflow(md$variables, md$variable_details)) + smoke <- spec$variables$smoke + expect_identical(smoke$missing_codes, c("6", "7", "8", "9")) + expect_equal(sum(smoke$missing_proportions[2:4]), sum(smoke$missing_proportions) - smoke$missing_proportions[1]) + expect_equal(smoke$missing_proportions[2], smoke$missing_proportions[3]) + expect_false(any(startsWith(smoke$missing_codes, "["))) +}) + +test_that("create_mock_data generates range missing codes on the v0.4 path (#58 reproduction)", { + md <- range_continuous_metadata() + expect_message( + result <- create_mock_data("study", md$variables, md$variable_details, + n = 3000, seed = 1, verbose = TRUE), + "Generating via v0.4 mock_spec pipeline" + ) + expect_true(all(c(997L, 998L, 999L) %in% result$age)) + valid <- result$age[result$age < 996] + expect_true(all(valid >= 18 & valid <= 85)) +}) + +test_that("categorical output never contains a literal bracketed code", { + md <- range_categorical_metadata() + result <- suppressWarnings(create_mock_data( + "study", md$variables, md$variable_details, n = 2000, seed = 1 + )) + expect_false(any(startsWith(as.character(result$smoke), "["))) + expect_true(all(c("7", "8", "9") %in% result$smoke)) +}) + +test_that("a non-integer range missing code fails with a message naming the fix", { + md <- range_continuous_metadata(missing_start = "[99.6,99.9]") + expect_error( + mock_spec_from_recodeflow(md$variables, md$variable_details), + "Variable 'age' has a range missing code '\\[99.6,99.9\\]' that is not an integer range" + ) +}) + +test_that("an unparseable bracketed missing code fails rather than passing through literally", { + md <- range_continuous_metadata(missing_start = "[996]") + expect_error( + mock_spec_from_recodeflow(md$variables, md$variable_details), + "has a range missing code '\\[996\\]' that is not an integer range" + ) +}) + +test_that("the minimal example's non-survival variables generate via the v0.4 pipeline", { + # The minimal example's Gompertz survival dates force the legacy generator, + # which hid that the rest of the example failed on the v0.4 path: BMI's + # "[997,999]" missing code (#58) and two fixture errors (BMI's garbage range + # typo, height's low-garbage proportion of 1.00). #40 depends on this. + vars <- system.file("extdata", "minimal-example", "variables.csv", package = "MockData") + dets <- system.file("extdata", "minimal-example", "variable_details.csv", package = "MockData") + if (!nzchar(vars) || !nzchar(dets)) skip("minimal-example fixtures not installed") + variables <- read.csv(vars, stringsAsFactors = FALSE, check.names = FALSE) + variable_details <- read.csv(dets, stringsAsFactors = FALSE, check.names = FALSE) + survival <- c("primary_event_date", "death_date", "ltfu_date", "admin_censor_date") + variables <- variables[!variables$variable %in% survival, ] + + expect_message( + result <- suppressWarnings(create_mock_data( + "minimal-example", variables, variable_details, + n = 500, seed = 1, verbose = TRUE + )), + "Generating via v0.4 mock_spec pipeline" + ) + expect_true(all(c("age", "smoking", "BMI", "height", "weight") %in% names(result))) + expect_true(any(result$BMI %in% c(997, 998, 999))) +}) diff --git a/tests/testthat/test-recodeflow-mock-spec.R b/tests/testthat/test-recodeflow-mock-spec.R index 0b0e797..853aeba 100644 --- a/tests/testthat/test-recodeflow-mock-spec.R +++ b/tests/testthat/test-recodeflow-mock-spec.R @@ -84,7 +84,7 @@ test_that("mock_spec_from_recodeflow preserves garbage and survival fields", { spec <- mock_spec_from_recodeflow(variables, variable_details) expect_equal(spec$variables$BMI$garbage_rules$low$proportion, 0.02) - expect_equal(spec$variables$BMI$garbage_rules$low$range, "[-10;15])") + expect_equal(spec$variables$BMI$garbage_rules$low$range, "[-10;15]") expect_equal(spec$variables$BMI$garbage_rules$high$proportion, 0.01) expect_equal(spec$variables$BMI$garbage_rules$high$range, "[60;150]") diff --git a/tests/testthat/test-simstudy-backend.R b/tests/testthat/test-simstudy-backend.R index a435c1f..2f7a8e1 100644 --- a/tests/testthat/test-simstudy-backend.R +++ b/tests/testthat/test-simstudy-backend.R @@ -251,3 +251,35 @@ test_that("generate_mock_data_simstudy keeps deferred formula variables loud", { "Formula evaluation is not yet implemented" ) }) + +test_that("generate_mock_data_simstudy skips genuine formula variables like the native backend, restoring the two-step workflow", { + # Task 2 review finding 2: the simstudy baseline path had no formula filter, + # so a genuine type = "formula" variable died with "Native backend does not + # support variable type 'formula'" inside .generate_native_only_baseline(). + skip_if_not_installed("simstudy") + + spec <- mock_spec( + mock_spec_continuous("height", range = c(1.4, 2.1)), + mock_spec_continuous("weight", range = c(45, 150)), + mock_spec_formula("bmi", formula = "weight / (height^2)") + ) + + baseline <- generate_mock_data_simstudy(spec, n = 20, seed = 11) + expect_false("bmi" %in% names(baseline)) + + result <- evaluate_mock_formulas(baseline, spec, seed = 11) + expect_identical(result$bmi, baseline$weight / (baseline$height^2)) +}) + +test_that(".native_only_variables() excludes formula variables regardless of simstudy availability", { + # Pins the fix's core logic in a helper that needs no simstudy install, per + # the review's request to pin it even where simstudy is absent. + spec <- mock_spec( + mock_spec_categorical("smoking", levels = c("never", "current")), + mock_spec_continuous("age_normal", range = c(18, 85), distribution = "normal", mean = 40, sd = 10), + mock_spec_formula("bmi", formula = "age_normal * 2") + ) + + result <- .native_only_variables(spec) + expect_setequal(names(result), "age_normal") +}) diff --git a/vignettes/advanced-topics.qmd b/vignettes/advanced-topics.qmd index b943cde..f67fab3 100644 --- a/vignettes/advanced-topics.qmd +++ b/vignettes/advanced-topics.qmd @@ -36,7 +36,7 @@ This guide covers advanced technical topics for MockData power users, including: ## Derived variables -Derived variables are calculated from other variables using custom functions. MockData identifies and skips derived variables during generation, leaving them for post-generation calculation. +Derived variables are calculated from other variables using custom functions. MockData identifies and skips derived variables during generation, leaving them for post-generation calculation — unless the variable also carries a `mockFormula` expression (v0.5+), in which case it is generated natively from that formula. See `vignette("design-philosophy-v04")` and `NEWS.md` for the formula-derived variables feature; the pattern below still applies to `DerivedVar::`/`Func::` variables without a `mockFormula`. ### Identifying derived variables diff --git a/vignettes/design-philosophy-v04.qmd b/vignettes/design-philosophy-v04.qmd index 8af0afe..9e25425 100644 --- a/vignettes/design-philosophy-v04.qmd +++ b/vignettes/design-philosophy-v04.qmd @@ -250,9 +250,12 @@ especially for `validate = FALSE`, `variable_details = NULL`, detail-level Several features are intentionally not solved in v0.4. -Formula-derived variables are detected and kept loud rather than silently -ignored. They need a dependency-aware evaluator, sandboxing rules, and clear -syntax. +Algebraic formula-derived variables landed in v0.5 (Phase A): a variable +carrying a `mockFormula` expression (e.g. `weight / (height^2)`) is generated +by evaluating that expression, in dependency order, over a restricted +sandbox exposing only the generated columns and a fixed allow-list of base +functions. `Func::` dispatch to a consuming package's own derivation +function (Phase B), and a more general evaluator, remain deferred. Multi-variable correlation and richer joint distributions are future work. `simstudy` is one possible engine for those features, but v0.4 does not claim to diff --git a/vignettes/reference-config.qmd b/vignettes/reference-config.qmd index 48bcb2c..cf13af1 100644 --- a/vignettes/reference-config.qmd +++ b/vignettes/reference-config.qmd @@ -1597,7 +1597,7 @@ bmi_fun <- function(height, weight) { mock_data$BMI_derived <- bmi_fun(mock_data$height, mock_data$weight) ``` -**Note:** Derived variables are NOT generated by create_mock_data(). See [Advanced topics: Derived variables](advanced-topics.html#derived-variables) for details. +**Note:** Derived variables are NOT generated by create_mock_data() unless they also carry a `mockFormula` expression (v0.5+); the example above has no `mockFormula`, so `BMI_derived` is excluded as shown. See [Advanced topics: Derived variables](advanced-topics.html#derived-variables) for details. ---