# tidy xgboost

    library(autostats)
    library(workflows)
    library(dplyr)
    library(tune)
    library(rsample)
    library(hardhat)
    library(broom.mixed)
    library(Ckmeans.1d.dp)
    library(igraph)

`autostats` provides convenient wrappers for modeling, visualizing, and predicting using a tidy workflow. The emphasis is on rapid iteration and quick results using an intuitive interface based off the `tibble` and `tidy_formula`.

# Prepare data

Set up the iris data set for modeling. Create dummies and any new columns before making the formula. This way the same formula can be use throughout the modeling and prediction process.

    set.seed(34)

     iris %>%
      dplyr::as_tibble() %>%
      framecleaner::create_dummies(remove_first_dummy  = TRUE) -> iris1
    #> 1 column(s) have become 2 dummy columns

     iris1 %>%
     tidy_formula(target = Petal.Length) -> petal_form

     petal_form
    #> Petal.Length ~ Sepal.Length + Sepal.Width + Petal.Width + species_versicolor +
    #>     species_virginica
    #> <environment: 0x130e95750>

Use the rsample package to split into train and validation sets.

    iris1 %>%
      rsample::initial_split() -> iris_split

    iris_split %>%
      rsample::analysis() -> iris_train

    iris_split %>%
      rsample::assessment() -> iris_val

    iris_split
    #> <Training/Testing/Total>
    #> <112/38/150>

# Fit boosting models and visualize

Fit models to the training set using the formula to predict `Petal.Length`. Variable importance using gain for each `xgboost` model can be visualized.

## xgboost with grid search hyperparameter optimization

`auto_tune_xgboost` returns a workflow object with tuned parameters and requires some postprocessing to get a trained `xgb.Booster` object like `tidy_xgboost`.

`xgboost` also can be tuned using a grid that is created internally using `dials::grid_max_entropy`. The `n_iter` parameter is passed to `grid_size`. Parallelization is highly effective in this method, so the default argument `parallel = TRUE` is recommended.

    iris_train %>%
      auto_tune_xgboost(formula = petal_form, n_iter = 5L,trees = 20L, loss_reduction = 2, mtry = 3, tune_method = "grid", parallel = FALSE, counts = TRUE) -> xgb_tuned_grid
    #> i Fold1: preprocessor 1/1
    #> i Fold1: preprocessor 1/1, model 1/1
    #> i Fold1: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold2: preprocessor 1/1
    #> i Fold2: preprocessor 1/1, model 1/1
    #> i Fold2: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold3: preprocessor 1/1
    #> i Fold3: preprocessor 1/1, model 1/1
    #> i Fold3: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold4: preprocessor 1/1
    #> i Fold4: preprocessor 1/1, model 1/1
    #> i Fold4: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold5: preprocessor 1/1
    #> i Fold5: preprocessor 1/1, model 1/1
    #> i Fold5: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold1: preprocessor 1/1
    #> i Fold1: preprocessor 1/1, model 1/1
    #> i Fold1: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold2: preprocessor 1/1
    #> i Fold2: preprocessor 1/1, model 1/1
    #> i Fold2: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold3: preprocessor 1/1
    #> i Fold3: preprocessor 1/1, model 1/1
    #> i Fold3: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold4: preprocessor 1/1
    #> i Fold4: preprocessor 1/1, model 1/1
    #> i Fold4: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold5: preprocessor 1/1
    #> i Fold5: preprocessor 1/1, model 1/1
    #> i Fold5: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold1: preprocessor 1/1
    #> i Fold1: preprocessor 1/1, model 1/1
    #> i Fold1: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold2: preprocessor 1/1
    #> i Fold2: preprocessor 1/1, model 1/1
    #> i Fold2: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold3: preprocessor 1/1
    #> i Fold3: preprocessor 1/1, model 1/1
    #> i Fold3: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold4: preprocessor 1/1
    #> i Fold4: preprocessor 1/1, model 1/1
    #> i Fold4: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold5: preprocessor 1/1
    #> i Fold5: preprocessor 1/1, model 1/1
    #> i Fold5: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold1: preprocessor 1/1
    #> i Fold1: preprocessor 1/1, model 1/1
    #> i Fold1: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold2: preprocessor 1/1
    #> i Fold2: preprocessor 1/1, model 1/1
    #> i Fold2: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold3: preprocessor 1/1
    #> i Fold3: preprocessor 1/1, model 1/1
    #> i Fold3: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold4: preprocessor 1/1
    #> i Fold4: preprocessor 1/1, model 1/1
    #> i Fold4: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold5: preprocessor 1/1
    #> i Fold5: preprocessor 1/1, model 1/1
    #> i Fold5: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold1: preprocessor 1/1
    #> i Fold1: preprocessor 1/1, model 1/1
    #> i Fold1: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold2: preprocessor 1/1
    #> i Fold2: preprocessor 1/1, model 1/1
    #> i Fold2: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold3: preprocessor 1/1
    #> i Fold3: preprocessor 1/1, model 1/1
    #> i Fold3: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold4: preprocessor 1/1
    #> i Fold4: preprocessor 1/1, model 1/1
    #> i Fold4: preprocessor 1/1, model 1/1 (predictions)
    #> i Fold5: preprocessor 1/1
    #> i Fold5: preprocessor 1/1, model 1/1
    #> i Fold5: preprocessor 1/1, model 1/1 (predictions)

    xgb_tuned_grid %>%
      parsnip::fit(iris_train) %>%
      parsnip::extract_fit_engine() -> xgb_tuned_fit_grid

    xgb_tuned_fit_grid %>%
      visualize_model()

## xgboost with default parameters

    iris_train %>%
      tidy_xgboost(formula = petal_form) -> xgb_base
    #> accuracy tested on a validation set
    #> # A tibble: 3 × 2
    #>   .metric .estimate
    #>   <chr>       <dbl>
    #> 1 ccc         0.857
    #> 2 rmse        0.798
    #> 3 rsq         0.816

## xgboost with hand-picked parameters

    iris_train %>%
      tidy_xgboost(petal_form,
                   trees = 250L,
                   tree_depth = 3L,
                   sample_size = .5,
                   mtry = .5,
                   min_n = 2) -> xgb_opt
    #> accuracy tested on a validation set
    #> # A tibble: 3 × 2
    #>   .metric .estimate
    #>   <chr>       <dbl>
    #> 1 ccc         0.960
    #> 2 rmse        0.460
    #> 3 rsq         0.948

# predict on validation set

## make predictions

Predictions are iteratively added to the validation data frame. The name of the column is automatically created using the models name and the prediction target.

    xgb_base %>%
      tidy_predict(newdata = iris_val, form = petal_form) -> iris_val2
    #> created the following column: Petal.Length_preds_xgb_base

    xgb_opt %>%
      tidy_predict(newdata = iris_val2, petal_form) -> iris_val3
    #> created the following column: Petal.Length_preds_xgb_opt

    iris_val3 %>%
      names()
    #> [1] "Sepal.Length"                "Sepal.Width"
    #> [3] "Petal.Length"                "Petal.Width"
    #> [5] "species_versicolor"          "species_virginica"
    #> [7] "Petal.Length_preds_xgb_base" "Petal.Length_preds_xgb_opt"

## predictions with eval_preds

Instead of evaluationg these prediction 1 by 1, This step is automated with `eval_preds`. This function is specifically designed to evaluate predicted columns with names given from `tidy_predict`.

    iris_val3 %>%
      eval_preds()
    #> # A tibble: 6 × 5
    #>   .metric .estimator .estimate model    target
    #>   <chr>   <chr>          <dbl> <chr>    <chr>
    #> 1 ccc     standard       0.976 xgb_base Petal.Length
    #> 2 ccc     standard       0.980 xgb_opt  Petal.Length
    #> 3 rmse    standard       0.373 xgb_base Petal.Length
    #> 4 rmse    standard       0.338 xgb_opt  Petal.Length
    #> 5 rsq     standard       0.971 xgb_base Petal.Length
    #> 6 rsq     standard       0.975 xgb_opt  Petal.Length

# get shapley values

`tidy_shap` has similar syntax to `tidy_predict` and can be used to get shapley values from `xgboost` models on a validation set.

    xgb_base %>%
      tidy_shap(newdata = iris_val, form = petal_form) -> shap_list

    shap_list$shap_tbl
    #> # A tibble: 38 × 6
    #>    Sepal.Length Sepal.Width Petal.Width species_versicolor species_virginica
    #>           <dbl>       <dbl>       <dbl>              <dbl>             <dbl>
    #>  1      -0.454      -0.0765       -1.46             0.0259            -0.127
    #>  2      -0.486      -0.0498       -1.46             0.0259            -0.124
    #>  3      -0.0130     -0.104        -1.43             0.0375            -0.139
    #>  4      -0.454      -0.0765       -1.46             0.0259            -0.127
    #>  5      -0.434      -0.0772       -1.48             0.0232            -0.125
    #>  6      -0.454      -0.0765       -1.46             0.0259            -0.127
    #>  7      -0.497      -0.0765       -1.38             0.0282            -0.127
    #>  8      -0.486      -0.0498       -1.46             0.0259            -0.124
    #>  9      -0.486      -0.0498       -1.46             0.0259            -0.124
    #> 10      -0.427       0.241        -1.40             0.0259            -0.126
    #> # ℹ 28 more rows
    #> # ℹ 1 more variable: `(Intercept)` <dbl>

    shap_list$shap_summary
    #> # A tibble: 6 × 5
    #>   name                  cor     var      sum sum_abs
    #>   <chr>               <dbl>   <dbl>    <dbl>   <dbl>
    #> 1 (Intercept)        NA     0       139.      139.
    #> 2 Petal.Width         0.957 1.21      6.86     39.2
    #> 3 Sepal.Length        0.959 0.166     1.23     13.0
    #> 4 species_virginica   0.994 0.0247    0.947     5.72
    #> 5 species_versicolor -0.893 0.00353   1.19      2.32
    #> 6 Sepal.Width        -0.826 0.00607  -0.0164    2.04

    shap_list$swarmplot

    shap_list$scatterplots

## understand xgboost with other functions from the original package

Overfittingin the base config may be related to growing deep trees.

     xgb_base %>%
      xgboost::xgb.plot.deepness()

     xgb_base %>%
      xgboost::xgb.plot.deepness()

Plot the first tree in the model. The small in terminal leaves suggests overfitting in the base model.

    xgb_base %>%
      xgboost::xgb.plot.tree(model = ., trees = 1)