Encode category metadata as reusable named sets

R/create-Rmd-table.R

@@ -1,4 +1,4 @@
-create_Rmd_table <- function(file_type, file_metadata, column_metadata, column_category_metadata) {
+create_Rmd_table <- function(file_type, file_metadata, column_metadata, category_set_metadata) {
   column_metadata_rows <- column_metadata[.matches_file_type(column_metadata$fileName, file_type), ]
   if(nrow(column_metadata_rows) == 0) {
     stop(glue::glue("No column metadata rows found for file {file_type}"))
@@ -15,10 +15,7 @@ create_Rmd_table <- function(file_type, file_metadata, column_metadata, column_c
     column_metadata_row <- column_metadata_rows[column_metadata_row_index, ]
 
     if(column_metadata_row$columnType == "category") {
-      column_categories_metadata_rows <- column_category_metadata[column_category_metadata$fileName == file_type & column_category_metadata$columnName == column_metadata_row$columnName, ]
-      if(nrow(column_categories_metadata_rows) == 0) {
-        stop(glue::glue("No column categories metadata rows found for file {file_type} for column {column_metadata_row$columnName}"))
-      }
+      column_categories_metadata_rows <- get_categories(column_metadata_row$categorySet, category_set_metadata)
       for(column_categories_metadata_rows_index in 1:nrow(column_categories_metadata_rows)) {
         column_categories_metadata_row <- column_categories_metadata_rows[column_categories_metadata_rows_index, ]
         if(column_categories_metadata_rows_index == 1) {

R/create-reference-Rmd.R

@@ -1,7 +1,7 @@
 create_reference_Rmd <- function(
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
   ) {
   reference_Rmd_lines <- c()
 
@@ -60,16 +60,14 @@ create_reference_Rmd <- function(
     )
     columns_table_rows <- purrr::pmap_chr(
      column_metadata_for_file,
-     function(columnName, columnType, columnLabel, ...) {
-        column_categories <- dplyr::if_else(
-          columnType == "category",
+     function(columnName, columnType, columnLabel, categorySet, ...) {
+        column_categories <- if (columnType == "category") {
           .format_column_categories(
-            column_category_metadata[
-              column_category_metadata$fileName == file_name &
-                column_category_metadata$columnName == columnName, ]
-          ),
+            get_categories(categorySet, category_set_metadata)
+          )
+        } else {
           ""
-        )
+        }
         columns_table_row <- paste(
           "|", columnName,
           "|", columnLabel,
@@ -98,7 +96,7 @@ create_reference_Rmd <- function(
 #'
 #' @param categories a data.frame containing the metadata for all
 #' the categories that need to be formatted. The structure should follow
-#' what's in the inst/metadata/column-category.csv file.
+#' what is returned by [get_categories()].
 #' @returns a string containing the formatted categories
 .format_column_categories <- function(categories) {
   if(nrow(categories) == 0) {

R/get-categories.R

@@ -0,0 +1,19 @@
+#' Get the category values for a category set(s)
+#'
+#' @param category_set A `categorySet` value from `column-metadata.csv`, for
+#' example `"variable-type"` or `"variable-type;type-start-na"`
+#' @param category_set_metadata The data frame read from `category-set.csv`
+#'
+#' @return A data frame with the columns `columnValue` and `desc`
+get_categories <- function(category_set, category_set_metadata) {
+  set_names <- strsplit(category_set, ";")[[1]]
+  categories <- category_set_metadata[
+    category_set_metadata$categorySet %in% set_names,
+    c("columnValue", "desc")
+  ]
+  if (nrow(categories) == 0) {
+    stop(glue::glue("No categories found for category set {category_set}"))
+  }
+  rownames(categories) <- NULL
+  categories
+}

R/validate-model-paramater-file.R

@@ -20,9 +20,9 @@ validate_model_parameters <- function(model_parameters_file_path) {
       system.file("metadata/column-metadata.csv", package = "model.parameters"),
       fileEncoding = "UTF-8-BOM"
     )
-  column_category_metadata <-
+  category_set_metadata <-
     read.csv(
-      system.file("metadata/column-category.csv", package = "model.parameters"),
+      system.file("metadata/category-set.csv", package = "model.parameters"),
       fileEncoding = "UTF-8-BOM"
     )
 
@@ -43,7 +43,7 @@ validate_model_parameters <- function(model_parameters_file_path) {
     basename(model_parameters_file_path),
     file_metadata,
     column_metadata,
-    column_category_metadata
+    category_set_metadata
   )
   if (length(model_parameters_file_errors) > 0) {
     stop(cat(model_parameters_file_errors, sep = "\n"))
@@ -74,7 +74,7 @@ validate_model_parameters <- function(model_parameters_file_path) {
       basename(model_parameter_file_path),
       file_metadata,
       column_metadata,
-      column_category_metadata
+      category_set_metadata
     )
     file_errors <- c(
       file_errors,
@@ -102,7 +102,7 @@ validate_model_parameters <- function(model_parameters_file_path) {
             basename(model_step_file_path),
             file_metadata,
             column_metadata,
-            column_category_metadata
+            category_set_metadata
           )
           return(validation)
         }
@@ -121,7 +121,7 @@ validate_file <- function(file,
                           file_name,
                           file_metadata,
                           column_metadata,
-                          column_category_metadata) {
+                          category_set_metadata) {
   current_file_metadata <- file_metadata[file_metadata$fileName == file_type,]
 
   if (nrow(current_file_metadata) == 0) {
@@ -172,16 +172,11 @@ validate_file <- function(file,
                }
              },
              "category" = {
-               column_categories_metadata <- column_category_metadata[column_category_metadata$fileName == file_type &
-                                                                        column_category_metadata$columnName == column_metadata_row$columnName,]
-               if (nrow(column_category_metadata) == 0) {
-                 stop(
-                   glue::glue(
-                     "No categories metadata found for file {file_type} for column {column_category_metadata$columnName}"
-                   )
-                 )
-               }
-
+               column_categories_metadata <- get_categories(
+                 column_metadata_row$categorySet,
+                 category_set_metadata
+               )
+
                column_values <- column_categories_metadata$columnValue
                if (!column_value %in% column_values) {
                  errors <- c(

docs/2-model-parameter-files.qmd

@@ -1,12 +1,13 @@
 ```{r echo=FALSE}
 source("../R/matches-file-type.R")
 source("../R/create-Rmd-table.R")
+source("../R/get-categories.R")
 ```
 
 ```{r read metadata files, echo=FALSE}
 file_metadata <- read.csv("../inst/metadata/file-metadata.csv", fileEncoding = "UTF-8-BOM")
 column_metadata <- read.csv("../inst/metadata/column-metadata.csv", fileEncoding = "UTF-8-BOM")
-column_category_metadata <- read.csv("../inst/metadata/column-category.csv", fileEncoding = "UTF-8-BOM")
+category_set_metadata <- read.csv("../inst/metadata/category-set.csv", fileEncoding = "UTF-8-BOM")
 ```
 
 # Model Parameter files
@@ -22,7 +23,7 @@ create_Rmd_table(
   "model-export",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -55,7 +56,7 @@ create_Rmd_table(
   "model-steps",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -113,7 +114,7 @@ create_Rmd_table(
   "dummy",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -156,7 +157,7 @@ create_Rmd_table(
   "center",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -188,7 +189,7 @@ create_Rmd_table(
   "rcs",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -236,7 +237,7 @@ create_Rmd_table(
   "interaction",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -283,7 +284,7 @@ create_Rmd_table(
   "beta-coefficients",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 
@@ -294,7 +295,7 @@ create_Rmd_table(
   "baseline-hazards",
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```
 

docs/5-reference.qmd

@@ -1,6 +1,7 @@
 ```{r, echo=FALSE, results='asis'}
 source("../R/matches-file-type.R")
 source("../R/create-reference-Rmd.R")
+source("../R/get-categories.R")
 
 file_metadata <- read.csv(
   "../inst/metadata/file-metadata.csv",
@@ -10,13 +11,13 @@ column_metadata <- read.csv(
   "../inst/metadata/column-metadata.csv",
   fileEncoding = "UTF-8-BOM"
 )
-column_category_metadata <- read.csv(
-  "../inst/metadata/column-category.csv",
+category_set_metadata <- read.csv(
+  "../inst/metadata/category-set.csv",
   fileEncoding = "UTF-8-BOM"
 )
 create_reference_Rmd(
   file_metadata,
   column_metadata,
-  column_category_metadata
+  category_set_metadata
 )
 ```

inst/metadata/README.md

@@ -2,7 +2,7 @@ The CSV files in this folder contains metadata for the different model
 parameter files, for example the types of files, their columns, their types
 etc.
 
-## Metadata Columns 
+## Metadata Columns
 
 Use camel case for metadata columns. For example, `fileName`, `columnName`, and
 `columnValue`.
@@ -14,16 +14,43 @@ Use kebab case for metadata column values, for example, `simple-model`,
 metadata column whose values should be camel case. In addition, human friendly
 language should be used for free form text metadata columns like descriptions
 and labels should be written in a human readable way, for example, the `desc`
-column in the `column-category.csv` file and the `fileDesc` column in the
+column in the `category-set.csv` file and the `fileDesc` column in the
 `file-metadata.csv` file.
 
 When the data type for a value is an array i.e. it needs to encode multiple
 values, a colon (;) should be used as the separator, for example, `a;b`. The
-`algorithmType` column in the `file-metadata.csv` file uses this format.
+`algorithmType` column in the `file-metadata.csv` file and the `categorySet`
+column in the `column-metadata.csv` file use this format.
+
+## Category Sets
+
+Categorical columns have a fixed set of allowed values called a
+**category set**. All category sets are defined in the `category-set.csv` file.
+
+Each row in this file defines a category using the following columns:
+
+- `categorySet`: The name of the category set that the category is part of;
+- `columnValue`: The category value; and
+- `desc`: The human readable description of the category.
+
+A category column then references its set(s) by name through the
+`categorySet` column in `column-metadata.csv`.
+
+Multiple categorical columns that have the same category values and
+descriptions should reuse the same category set. For example, the
+`variable-type` set defines the `cat` and `cont` values once, and every column
+that accepts those values (such as `typeEnd` and the various `*VariableType`
+columns) references it. Conversely, columns that share a value but give it a
+different meaning (e.g., the `N/A` in `model-steps's` `fileType` versus
+`type-start-na`) should keep separate sets so each keeps its own description.
+
+A `categorySet` value may list more than one set, separated by a semi-colon, in
+which case the column accepts the union of those sets' values. For example,
+`variable-type;type-start-na` allows `cat`, `cont`, and `N/A`.
 
 ## New Algorithms
 
 When adding a newly supported algorithm to the repository, make sure to update
-the `algorithmType` column in the `file-metadata.csv` appropriately. This 
-column is used to specify which algorithm(s) each file is applicable to. Use a 
+the `algorithmType` column in the `file-metadata.csv` appropriately. This
+column is used to specify which algorithm(s) each file is applicable to. Use a
 value of `all` if a file is applicable to all algorithms.

inst/metadata/category-set.csv

@@ -0,0 +1,41 @@
+categorySet,columnValue,desc
+variable-type,cat,Categorical type
+variable-type,cont,Continuous type
+variable-type-full,Categorical,Categorical type
+variable-type-full,Continuous,Continuous type
+type-start-na,N/A,Used mainly for derived variables when there are many start variables
+model-export-file-type,variables,A variables file
+model-export-file-type,variable-details,A variable details file
+model-export-file-type,descriptive,A descriptive lookup file
+model-export-file-type,descriptive-bins,A descriptive bins file
+model-export-file-type,model-export,A model export file
+model-export-file-type,model-steps,A model steps
+model-export-file-type,dummy,A dummy file
+model-export-file-type,center,A centering file
+model-export-file-type,rcs,An RCS file
+model-export-file-type,interaction,An interaction file
+model-export-file-type,fine-and-gray,A fine and gray file
+model-export-file-type,cox,A cox file
+model-export-file-type,survival_bins,File with survival data for the different bins
+model-export-file-type,lookup,Contains the range of score values for each biin in a survival algorithm
+model-export-file-type,validate,Contains the validation rules for variables in the algorithm
+model-export-file-type,tables,Defines a tables model export file
+model-step,dummy,Step used to create dummy variables
+model-step,center,Step used to create centered variables
+model-step,rcs,Steps used to create spline terms
+model-step,interaction,Step used to create interaction terms
+model-step,fine-and-gray,Step used to calculate the outcome of a fine and gray model
+model-step,cox,Step used to calculate the outcome of a cox proportional hazards model
+model-step,simple-model,Step used to calculate the outcome of a simple model
+model-step,logistic-regression,Step used to calculate the outcome of a logistic regression model
+model-steps-file-type,N/A,Missing since certain steps don't need to specify the file type
+model-steps-file-type,beta-coefficients,A beta coefficients file for a fine and gray model or a cox model
+model-steps-file-type,baseline-hazards,The baseline hazards for a fine and gray model or a cox model
+validation-rule,type,Validate the data type of the variable
+validation-rule,range,Validate that the variable is within a range of values. Meant for continuous variables.
+validation-rule,allowed,Validate that the variable contains a value from a set of valid values. Meant for categorical variables.
+validation-rule,nullable,Whether missing values are allowed for a variable
+error-handle,error,"Failed validations for the variable should throw an error, stopping the scoring process"
+error-handle,warning,Failed validations for the variable should log a warning and continue the scoring process
+error-handle,truncate,"Failed validations for the variable should log a warning, truncate the failed variable value, and continue scoring process"
+simple-model-name,outputVariableName,The name of the output variable for the model. Should be defined in the variables and variable details sheets.