---
title: "LUCID Serial Tutorial: 6 Consecutive Parallel Stages with Monotone Missingness"
output: html_document
---

## 1) Why This Tutorial

This tutorial demonstrates a stress-test architecture that mirrors a real production pattern:

- **LUCID in serial** with **6 consecutive stages**
- each stage is a **parallel submodel with 2 omics layers**
- **K = 2** for every layer in every stage
- **no penalties** (`Rho_G = Rho_Z_Mu = Rho_Z_Cov = 0`)

Missing-data design in this tutorial:

- stage-wise **listwise missingness increases** with stage
- once a subject is listwise-missing at stage `s`, it stays missing at all later stages (monotone pattern)

Goal: verify this design runs with the current missing-data mechanism and inspect stage-wise missingness summaries.

## 2) Setup

```{r setup, message=FALSE, warning=FALSE}
required_pkgs <- c("mclust", "glmnet", "glasso", "nnet", "boot", "progress")
missing_pkgs <- required_pkgs[!vapply(required_pkgs, requireNamespace, logical(1), quietly = TRUE)]
if (length(missing_pkgs) > 0) {
  stop("Please install required packages first: ", paste(missing_pkgs, collapse = ", "))
}

library(mclust)
library(glmnet)
library(glasso)
library(nnet)
library(boot)
library(progress)

locate_pkg_root <- function() {
  cands <- c(
    file.path(getwd(), "LUCIDus-3.0.3"),
    file.path(getwd(), "..", "LUCIDus-3.0.3"),
    getwd()
  )
  cands <- unique(normalizePath(cands, winslash = "/", mustWork = FALSE))
  for (cand in cands) {
    if (file.exists(file.path(cand, "R", "EM_all.R")) &&
        file.exists(file.path(cand, "data", "simulated_HELIX_data.rda"))) {
      return(cand)
    }
  }
  stop("Could not find package root")
}

pkg_root <- locate_pkg_root()
r_files <- list.files(file.path(pkg_root, "R"), pattern = "[.]R$", full.names = TRUE)
invisible(lapply(r_files, source))
```

## 3) Simulate Associated Data (Not Independent Noise)

Key design choices:

1. `G`, `CoG`, `CoY` are correlated.
2. Stage latent signals are serially dependent (`stage s` depends on `stage s-1`).
3. Each stage-layer `Z` is generated from latent stage signal + exposures + shared component.
4. `Y` is associated with final-stage latent structure and exposure/covariates.

```{r simulate-associated-data}
make_serial_six_parallel_data <- function(n = 60, pG = 5, pZ = 3, seed = 5252) {
  set.seed(seed)

  # Exposure block
  G <- matrix(rnorm(n * pG), nrow = n, ncol = pG)
  colnames(G) <- paste0("G", seq_len(pG))

  # Covariates linked to exposures (induced association)
  CoG <- cbind(
    cov1 = G[, 1] + rnorm(n, sd = 0.25),
    cov2 = G[, 2] - 0.5 * G[, 3] + rnorm(n, sd = 0.25)
  )
  CoY <- CoG

  # Serial latent signals
  eta <- matrix(0, nrow = n, ncol = 6)
  x <- matrix(0, nrow = n, ncol = 6)

  eta[, 1] <- 0.9 * G[, 1] - 0.7 * G[, 2] + 0.4 * CoG[, 1] + rnorm(n, sd = 0.5)
  x[, 1] <- as.numeric(eta[, 1] > median(eta[, 1]))

  for (s in 2:6) {
    eta[, s] <- 0.7 * as.numeric(scale(eta[, s - 1])) +
      0.5 * G[, 1] - 0.4 * G[, 3] + 0.35 * x[, s - 1] + rnorm(n, sd = 0.6)
    x[, s] <- as.numeric(eta[, s] > median(eta[, s]))
  }

  # 6 stages; each stage has 2 parallel layers
  Z <- vector("list", 6)
  names(Z) <- paste0("stage", seq_len(6))

  for (s in seq_len(6)) {
    shared <- rnorm(n, sd = 0.35)

    layer1 <- cbind(
      1.2 * x[, s] + 0.5 * G[, 1] + shared + rnorm(n, sd = 0.45),
      0.9 * x[, s] - 0.3 * G[, 2] + shared + rnorm(n, sd = 0.45),
      0.7 * x[, s] + 0.4 * G[, 4] + shared + rnorm(n, sd = 0.45)
    )

    layer2 <- cbind(
      -1.0 * x[, s] + 0.45 * G[, 2] + shared + rnorm(n, sd = 0.45),
      -0.8 * x[, s] - 0.35 * G[, 1] + shared + rnorm(n, sd = 0.45),
      -0.6 * x[, s] + 0.30 * G[, 5] + shared + rnorm(n, sd = 0.45)
    )

    colnames(layer1) <- paste0("s", s, "_L1_f", seq_len(pZ))
    colnames(layer2) <- paste0("s", s, "_L2_f", seq_len(pZ))

    Z[[s]] <- list(layer1 = layer1, layer2 = layer2)
  }

  # Outcome associated with final latent stage + exposure/covariate terms
  Y <- 0.8 * x[, 6] + 0.45 * G[, 1] - 0.25 * G[, 2] + 0.35 * CoY[, 2] + rnorm(n, sd = 0.7)

  list(G = G, Z = Z, Y = as.numeric(Y), CoG = CoG, CoY = CoY)
}

d <- make_serial_six_parallel_data(seed = 5252)
str(d, max.level = 2)
```

## 4) Inject Monotone Stage-Wise Missingness

Policy encoded here:

- listwise-missing row counts by stage: `0, 4, 8, 12, 16, 20`
- if subject `i` is listwise missing at stage `s`, then subject `i` is also listwise missing at all later stages
- add light sporadic missingness that also increases with stage

```{r monotone-missingness}
apply_monotone_missingness <- function(d, pZ = 3) {
  listwise_n_by_stage <- c(0, 4, 8, 12, 16, 20)

  for (s in seq_len(6)) {
    if (listwise_n_by_stage[s] > 0) {
      miss_rows <- seq_len(listwise_n_by_stage[s])
      d$Z[[s]]$layer1[miss_rows, ] <- NA
      d$Z[[s]]$layer2[miss_rows, ] <- NA
    }

    spor_rows <- seq_len(min(nrow(d$G), s + 1))
    d$Z[[s]]$layer1[spor_rows, 1] <- NA
    d$Z[[s]]$layer2[spor_rows, pZ] <- NA
  }

  d$listwise_n_by_stage <- listwise_n_by_stage
  d
}

d_miss <- apply_monotone_missingness(d)
d_miss$listwise_n_by_stage
```

## 5) Fit Serial Model (6 Parallel Stages, K = 2 per Layer)

Notes:

- no penalties (`Rho_* = 0`)
- stage topology is `list(list(2,2), ..., list(2,2))` with length 6

```{r fit-serial-6parallel, warning=FALSE}
K6 <- replicate(6, list(2, 2), simplify = FALSE)

set.seed(5252)
serial_fit <- estimate_lucid(
  lucid_model = "serial",
  G = d_miss$G,
  Z = d_miss$Z,
  Y = d_miss$Y,
  CoG = d_miss$CoG,
  CoY = d_miss$CoY,
  family = "normal",
  K = K6,
  Rho_G = 0,
  Rho_Z_Mu = 0,
  Rho_Z_Cov = 0,
  max_itr = 6,
  max_tot.itr = 36,
  tol = 1e-2,
  seed = 5252
)

class(serial_fit)
length(serial_fit$submodel)
sapply(serial_fit$submodel, class)
```

## 6) Verify Stage-Wise Missingness Summary

We check that recorded listwise row counts match the monotone design.

```{r check-missing-summary}
listwise_l1 <- vapply(serial_fit$missing_summary$stage, function(ms) {
  as.integer(ms$layer_summary$listwise_rows[1])
}, integer(1))

listwise_l2 <- vapply(serial_fit$missing_summary$stage, function(ms) {
  as.integer(ms$layer_summary$listwise_rows[2])
}, integer(1))

data.frame(
  stage = 1:6,
  expected_listwise = d_miss$listwise_n_by_stage,
  observed_layer1 = listwise_l1,
  observed_layer2 = listwise_l2
)

stopifnot(all(listwise_l1 == d_miss$listwise_n_by_stage))
stopifnot(all(listwise_l2 == d_miss$listwise_n_by_stage))
stopifnot(all(diff(listwise_l1) >= 0))
```

## 7) Confirm Final-Stage Numerical Stability

Even with aggressive later-stage missingness, final-stage posterior probabilities should remain finite.

```{r check-final-stage}
last_stage <- serial_fit$submodel[[6]]

all(is.finite(last_stage$inclusion.p[[1]]))
all(is.finite(last_stage$inclusion.p[[2]]))

stopifnot(all(is.finite(last_stage$inclusion.p[[1]])))
stopifnot(all(is.finite(last_stage$inclusion.p[[2]])))
```

## 8) User-Facing Summary Output

```{r print-summary, warning=FALSE}
serial_sum <- summary_lucid(serial_fit)
print(serial_sum)
```

## 9) Bootstrap Inference (Serial Model)

Because this model is already fit with zero penalties, we can directly run bootstrap CI.

Practical note:

- this architecture is relatively heavy (6 serial stages, each stage parallel),
- so we use a small `R` in the tutorial for runtime.
- in analysis runs, increase `R` for more stable intervals.

```{r serial-bootstrap, warning=FALSE}
set.seed(5253)

serial_boot <- boot_lucid(
  G = d_miss$G,
  Z = d_miss$Z,
  Y = d_miss$Y,
  lucid_model = "serial",
  CoG = d_miss$CoG,
  CoY = d_miss$CoY,
  model = serial_fit,
  R = 3,
  conf = 0.90
)

# Stage-wise bootstrap objects are returned under $stage
length(serial_boot$stage)
names(serial_boot$stage[[1]])
```

## 10) Summary With Bootstrap CI

This summary prints the same model tables plus CI columns for supported parameter blocks.

```{r serial-summary-bootstrap, warning=FALSE}
serial_sum_boot <- summary_lucid(serial_fit, boot.se = serial_boot)
print(serial_sum_boot)
```

## 11) Interpretation Notes for Real Data Use

- This run confirms the architecture executes with monotone missingness and stage-wise increasing listwise NA.
- In your real data, if a late stage has too many all-missing rows, initialization may become unstable; increase `N` or reduce missingness severity if needed.
- Keep this as a smoke/regression template for your real 6-stage serial-parallel pipeline.

## 12) Session Info

```{r}
sessionInfo()
```
