Summary Functions

Per-cluster Summary

The cluster_summary() function creates a data summary table stratified by missingness cluster. The function builds on gtsummary::tbl_summary(), so gtsummary-like statistics can be used for summarizing variables ( see tbl_summary() documentation for details ).

library(tidyverse)
library(reticulate)
library(rCISSVAE)
library(kableExtra)
library(gtsummary)

data(df_missing)
data(clusters)

## Integer clusters must be passed in as a factor
cluster_summary(data = df_missing, factor(clusters$clusters), 
include = setdiff(names(df_missing), "index"), 
statistic = list(
  all_continuous() ~ "{mean} ({sd})",
  all_categorical() ~ "{n} / {N}\n ({p}%)"), 
  missing = "always")

Missingness Heatmap

cluster_heatmap(
  data = df_missing, 
  clusters = paste0("Cluster ", clusters$clusters), ## Adds 'Cluster' to the cluster label
  cols_ignore = "index", 
  observed_color = "#23013aff", ## A dark purple
  missing_color = "yellow")

## `use_raster` is automatically set to TRUE for a matrix with more than
## 2000 columns You can control `use_raster` argument by explicitly
## setting TRUE/FALSE to it.
## 
## Set `ht_opt$message = FALSE` to turn off this message.

## 'magick' package is suggested to install to give better rasterization.
## 
## Set `ht_opt$message = FALSE` to turn off this message.

By-cluster imputation loss function

After running the model, you can get the per-cluster validation set imputation loss using the performance_by_cluster() function. Set ‘return_validation_dataset = TRUE’ in the run_cissvae() function to be able to use performance_by_cluster on the result object. If the validation dataset (val_data in result object) and imputed validation dataset (val_imputed in the result object) are not returned, the imputation loss cannot be calculated.

If the run_cissvae() function was used to generate clusters, set return_clusters=TRUE and the clusters will be part of the return object. Otherwise, use the ‘clusters’ parameter in performance_by_cluster() to input the clusters.

result = run_cissvae(
  data = df_missing,
  index_col = "index",
  val_proportion = 0.1, ## pass a vector for different proportions by cluster
  columns_ignore = c("Age", "Salary", "ZipCode10001", "ZipCode20002", "ZipCode30003"), ## If there are columns in addition to the index you want to ignore when selecting validation set, list them here. In this case, we ignore the 'demographic' columns because we do not want to remove data from them for validation purposes. 
  clusters = clusters$clusters, ## we have precomputed cluster labels so we pass them here
  epochs = 5,
  return_silhouettes = FALSE,
  return_history = TRUE,  # Get detailed training history
  verbose = FALSE,
  return_model = TRUE, ## Allows for plotting model schematic
  device = "cpu",  # Explicit device selection
  layer_order_enc = c("unshared", "shared", "unshared"),
  layer_order_dec = c("shared", "unshared", "shared"),
  return_validation_dataset = TRUE
)

cat(paste("Check necessary returns:", paste0(names(result), collapse = ", ")))

## Check necessary returns: imputed_dataset, model, training_history, val_data, val_imputed

performance_by_cluster(res = result, 
  group_col = NULL, 
  clusters = clusters$clusters,
  feature_cols = NULL, ## default, all numeric columns excluding group_col & cols_ignore
  by_group = FALSE,
  by_cluster = TRUE,
  cols_ignore =  c( "index", "Age", "Salary", "ZipCode10001", "ZipCode20002", "ZipCode30003") ## columns to not score
  )

## $overall
##     metric     n
## 1 90.99783 13783
## 
## $per_cluster
##   cluster mean_imputation_loss    n
## 1       0             48.70336 3408
## 2       1             96.68600 1787
## 3       2             80.57138 5000
## 4       3            142.86713 3588
## 
## $per_feature_overall
##    feature       type mean_imputation_loss   n
## 1      Y11 continuous            41.565153 486
## 2      Y12 continuous            89.328866 486
## 3      Y13 continuous           119.401816 488
## 4      Y14 continuous           160.124733 486
## 5      Y15 continuous           252.965269 484
## 6      Y21 continuous            66.576460 485
## 7      Y22 continuous           137.619200 489
## 8      Y23 continuous           127.223039 489
## 9      Y24 continuous           202.076766 484
## 10     Y25 continuous           234.807206 488
## 11     Y31 continuous            47.142491 592
## 12     Y32 continuous            87.786836 593
## 13     Y33 continuous           146.073318 597
## 14     Y34 continuous           168.344495 593
## 15     Y35 continuous           205.363351 593
## 16     Y41 continuous             7.179157 596
## 17     Y42 continuous            18.288482 596
## 18     Y43 continuous            37.223463 597
## 19     Y44 continuous            47.033124 597
## 20     Y45 continuous            61.172087 590
## 21     Y51 continuous             5.400970 591
## 22     Y52 continuous            19.634923 595
## 23     Y53 continuous            24.613987 601
## 24     Y54 continuous            26.400094 594
## 25     Y55 continuous            37.428103 593