# sim-validation

    library("splithalf")
    library("tidyr")

# Validaion Sim for splithalf

This script runs splithalf with many possible combinations of the options. Its main use is to ensure that the splithalf function

# Splithalf()

    # Thanks to Nathaniel Haines for the suggestion and the starting code (https://github.com/sdparsons/splithalf/issues/9).

    set.seed(01042021)

    n_participants <- 60 ## sample size
    n_trials <- 80
    n_blocks <- 2

    sim_data <- data.frame(participant_number = rep(1:n_participants,
                                                    each = n_blocks * n_trials),
                           trial_number = rep(1:n_trials,
                                              times = n_blocks * n_participants),
                           block_name = rep(c("A","B"),
                                            each = n_trials,
                                            length.out = n_participants * n_trials * n_blocks),
                           trial_type = rep(c("congruent","congruent",
                                              "incongruent","incongruent"),
                                            length.out = n_participants * n_trials * n_blocks / 2),
                           RT = rnorm(n_participants * n_trials * n_blocks,
                                      500,
                                      200),
                           ACC = 1)

    sim_data_acc <- data.frame(participant_number = rep(1:n_participants,
                                                        each = n_blocks * n_trials),
                           trial_number = rep(1:n_trials,
                                              times = n_blocks * n_participants),
                           block_name = rep(c("A","B"),
                                            each = n_trials,
                                            length.out = n_participants * n_trials * n_blocks),
                           trial_type = rep(c("congruent","congruent",
                                              "incongruent","incongruent"),
                                            length.out = n_participants * n_trials * n_blocks / 2),
                           RT = rnorm(n_participants * n_trials * n_blocks,
                                      500,
                                      200),
                           ACC = c(rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .5),
                                   rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .7),
                                   rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .9),
                                   rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .5),
                                   rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .7),
                                   rbinom(n_participants *
                                        n_trials *
                                        n_blocks / 6,
                                        1, .9)))

    params <- list(score = c("average",
                             "difference",
                             "difference_of_difference"),
                   halftype = c("oddeven",
                                "halfs",
                                "random"),
                   averae = c("mean",
                              "median") )

    expand.grid(params) %>%
      dplyr::arrange(., score, halftype)

## Response times

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data,
              outcome = "RT",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data,
              outcome = "RT",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

## Average accuracy rates

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",

              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "average",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "oddeven",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "halfs",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median")

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "mean",
              plot = TRUE)

    splithalf(data = sim_data_acc,
              outcome = "accuracy",
              score = "difference_of_difference",
              conditionlist = c("A", "B"),
              halftype = "random",
              permutations = 500,
              var.RT = "RT",
              var.condition = "block_name",
              var.participant = "participant_number",
              var.compare = "trial_type",
              var.ACC = "ACC",
              compare1 = "congruent",
              compare2 = "incongruent",
              average = "median",
              plot = TRUE)

# multiverse

Runs several multiverse

    set.seed(01042021)

    n_participants <- 80 ## sample size
    n_trials <- 120
    n_blocks <- 2

    sim_data_mv <- data.frame(participant_number = rep(1:n_participants,
                                                        each = n_blocks * n_trials),
                               trial_number = rep(1:n_trials,
                                                  times = n_blocks * n_participants),
                               block_name = rep(c(1,2),
                                                each = n_trials,
                                                length.out = n_participants * n_trials * n_blocks),
                               trial_type = rep(c("congruent","congruent",
                                                  "incongruent","incongruent"),
                                                length.out = n_participants * n_trials * n_blocks / 2),
                               RT = rnorm(n_participants * n_trials * n_blocks,
                                          500,
                                          200),
                               ACC = c(rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .5),
                                       rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .7),
                                       rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .9),
                                       rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .5),
                                       rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .7),
                                       rbinom(n_participants *
                                                n_trials *
                                                n_blocks / 6,
                                              1, .9)))

    specifications <- list(
      ACC_cutoff = c(0, 0.5),
      RT_min           = c(0, 200),
      RT_max            = c(2000, 3000),
      RT_sd_cutoff      = c(0, 2),
      split_by          = c("subject", "trial"),
      averaging_method  = c("mean")
    )

## internal consistency

    mv_splithalf <- splithalf(data = sim_data,
                            outcome = "RT",
                            score = "difference",
                            conditionlist = c("A"),
                            halftype = "random",
                            permutations = 50,
                            var.RT = "RT",
                            var.condition = "block_name",
                            var.participant = "participant_number",
                            var.compare = "trial_type",
                            var.ACC = "ACC",
                            compare1 = "congruent",
                            compare2 = "incongruent",
                            average = "mean")

    multiverse <- splithalf.multiverse(input = mv_splithalf,
                                       specifications = specifications)

    multiverse.plot(multiverse = multiverse,
                    title = "sim_multiverse")

## test-retest

    icc2 <- testretest.multiverse(data = sim_data_acc,
                           specifications,
                           test = "ICC2",

                           score = "difference",

                           var.participant = "participant_number",
                           var.ACC = "ACC",
                           var.RT = "RT",
                           var.time = "block_name",
                           var.compare = "trial_type",
                           compare1 = "congruent",
                           compare2 = "incongruent")

    multiverse.plot(icc2)

    icc3 <- testretest.multiverse(data = sim_data_acc,
                           specifications,
                           test = "ICC3",

                           score = "difference",

                           var.participant = "participant_number",
                           var.ACC = "ACC",
                           var.RT = "RT",
                           var.time = "block_name",
                           var.compare = "trial_type",
                           compare1 = "congruent",
                           compare2 = "incongruent")

    multiverse.plot(icc3)

    retest_cor <- testretest.multiverse(data = sim_data_acc,
                           specifications,
                           test = "cor",

                           score = "difference",

                           var.participant = "participant_number",
                           var.ACC = "ACC",
                           var.RT = "RT",
                           var.time = "block_name",
                           var.compare = "trial_type",
                           compare1 = "congruent",
                           compare2 = "incongruent")

    multiverse.plot(retest_cor)