---
title: "Report generated by the iraceplot package"
output: 
  html_document:
    toc: true
    highlight: pygments
    toc_float: true
---
```{css, echo=FALSE}
pre {
  white-space:pre !important;
  overflow-x: auto;
  max-width: 100%;
}
pre code {
  white-space:pre !important;
  overflow-x: auto;
}
.overflow {
  white-space:pre !important;
  overflow-x: auto;
  max-width: 100%;
}
```

```{r, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

```{r setup, include= FALSE}
library(iraceplot)
library(tidyr, quietly = TRUE)
library(dplyr, quietly = TRUE)
sections <- get0("sections", ifnotfound=list(experiments_matrix=TRUE,convergence=TRUE))
if (!exists("interactive_plots")) interactive_plots <- base::interactive()
# FIXME: Move show_table to the package so people can use it to create their
# own reports.
show_table <- function(x, row.names = FALSE, search = FALSE, lengthMenu = NULL,
                       searchHighlight = TRUE, dom='Blrtip', colorbar = NULL, style = list(), ...) {

  if (!interactive_plots) return(knitr::kable(x, row.names = row.names))

  if (search) dom <- sub("r", "fr", dom, fixed = TRUE)
  if (is.null(lengthMenu)) {
    pageLength <- NULL
  } else {
    pageLength <- lengthMenu[1]
    if (nrow(x) <= pageLength) dom <- gsub("[lip]", "", dom) # autoHideNavigation
  }
  columnDefs <- list(list(className = 'dt-right', targets = "_all"))
  tab <- DT::datatable(x, rownames = row.names,
                       class = 'compact row-border hover',
                       autoHideNavigation = TRUE,
                       extensions = c('Buttons', 'KeyTable'),
                       options = list(
                         keys = TRUE,
                         searchHighlight = search,
                         scrollX = TRUE,
                         pageLength=pageLength,
                         lengthMenu = lengthMenu,
                         dom = dom,
                         buttons = c('copy', 'csv'), #, 'colvis'),
                         columnDefs = columnDefs)) %>% DT::formatStyle(colnames(x), fontSize='85%')
  for (col in colorbar) {
    if(!(col %in% colnames(x)))
      stop("Column ", col, " not found in table: ", paste0(colnames(x), collapse=", "))
    if (all(is.na(x[, col]))) next # Don't add colorbar if all are NA.
    tab <- DT::formatStyle(tab, columns = col,
                           background = DT::styleColorBar(range(x[, col], na.rm=TRUE), 'lightblue'),
                           backgroundSize = '98% 88%',
                           backgroundRepeat = 'no-repeat',
                           backgroundPosition = 'center')
  }
  for (s in style) {
    tab <- do.call(DT::formatStyle, c(table=tab, s))
  }
  tab
}

best_elites <- as.character(irace_results$allElites[[length(irace_results$allElites)]])
irs <- irace_summarise(irace_results)
parameters <- irace_results$scenario$parameters
```

# Scenario
<details>
<summary>Click to show</summary>
```{r, echo=TRUE,eval=FALSE,code=capture.output(irace::printScenario(irace_results$scenario))}
```
</details> 


# Parameters

```{r parameters_summarise, echo=FALSE, results='asis'}
knitr::kable(parameters_summarise(parameters))
```
## Parameters tree
<pre>
```{r parameters_tree, echo=FALSE, results='asis'}
parameters_tree(parameters)
```
</pre>

## Parameters file
<details>
<summary>Click to show</summary>
<pre>
```{r printParameters, echo=FALSE, results='asis'}
irace::printParameters(parameters)
```
</pre>
</details> 


# General information
- irace version: `r {irs$version}`
- Iterations: `r {irs$n_iterations}`
- Configurations: `r {irs$n_configurations}`
- Instances: `r {irs$n_instances}`
- Experiments: `r {irs$n_experiments}`
- Elite configurations: `r {irs$n_elites}`
- Soft restarts: `r {irs$n_soft_restarts}`
- Rejected configurations: `r {irs$n_rejected}`
- Running time (seconds): `r {if (is.na(irs$time_wallclock)) "Unknown" else paste0(round(irs$time_cpu_user,0), " (CPU-user); ", round(irs$time_cpu_sys,0), " (CPU-sys); ", round(irs$time_wallclock,0), " (Wall-clock)")}`
- Termination reason: `r {irs$termination_reason}`

## By iteration

```{r table-by-iterations,fig.align="center", echo=FALSE}
show_table(summarise_by_iteration(irace_results), lengthMenu = c(20, 50, 100),
           colorbar = c("configurations", "instances", "experiments", "elites"))
```

## By instance

```{r table-by-instance,fig.align="center", echo=FALSE}
show_table(summarise_by_instance(irace_results), lengthMenu = c(20, 50, 100), search = TRUE,
           colorbar = c("experiments", "mean", "sd", "median", "min", "max"))
#columnDefs = list(list(targets = -1, searchable = FALSE))
```



# Elite configurations

The final best configurations found by irace are:

```{r table-elite,fig.align="center", echo=FALSE}
show_table(irace_results$allConfigurations[best_elites, , drop=FALSE],
           lengthMenu = c(10,20),
           colorbar = names(which(parameters$types %in% c("i", "r") & !parameters$isFixed)))
```

## Parallel coordinates visualization (only elites)

```{r parcoords-elites,fig.align="center", out.width="100%", echo=FALSE}
parallel_coord(irace_results)
```

# Sampling model

The frequency of the parameter values sampled by irace:

```{r fig-sampling-model,fig.align="center", out.width="100%", echo=FALSE}
sampling_frequency(irace_results)
```


# Testing performance

- Number of elites tested: `r {irace_results$scenario$testNbElites}`
- Iteration elites tested: `r {irace_results$scenario$testIterationElites}`

## Performance of the elite configurations on the test instances

```{r,fig.align="center", echo=FALSE}
if (has_testing_data(irace_results)) {
  best_results <- summarise_by_configuration(irace_results, instances = "test")
  show_table(best_results, lengthMenu = c(5, 10, 50),
             colorbar = c("mean", "sd", "median", "min", "max"))
} else {
  cat("No test instances given.\n")
}
```

## Final elite configurations on the test instances

```{r boxplot-test-elites-rpd,fig.align="center", out.width="100%", echo=FALSE}
if (has_testing_data(irace_results)) {
  boxplot_test(irace_results, type = "best", interactive = interactive_plots)
} else {
  cat("No test instances given.\n")
}
```
```{r boxplot-test-elites-raw,fig.align="center", out.width="100%", echo=FALSE}
if (has_testing_data(irace_results))
  boxplot_test(irace_results, type = "best", rpd = FALSE, interactive = interactive_plots)
```

## Iteration elite configurations  on the test instances

```{r boxplot-test-iteration-elites-rpd,fig.align="center", out.width="100%", echo=FALSE}
if (irace_results$scenario$testIterationElites && has_testing_data(irace_results)) {
   boxplot_test(irace_results, type = "all", show_points=FALSE, interactive = interactive_plots)
} else{
  cat("Iteration elites were not tested.\n")
}
```

# Training performance 

## Performance of the final elite configurations on the training instances

```{r table-train-elites,fig.align="center", echo=FALSE}
best_results <- summarise_by_configuration(irace_results, elites_only = TRUE, instances = "train")
show_table(best_results, lengthMenu = c(10, 20, 50, 100),
           colorbar = c("mean", "sd", "median", "min", "max", "rank_mean", "rank_sd"))
```

## Final elite configurations on the training instances


```{r boxplot-train-elite-rpd,fig.align="center", out.width="100%", echo=FALSE}
boxplot_training(irace_results, interactive = interactive_plots)
```
```{r boxplot-train-elite-raw,fig.align="center", out.width="100%", echo=FALSE}
boxplot_training(irace_results, rpd = FALSE, interactive = interactive_plots)
```

```{asis, echo=irace_results$scenario$testIterationElites}
## Iteration elite configurations on the training instances
```
```{r boxplot-train-iteration-elites-rpd,fig.align="center", out.width="100%", echo=FALSE}
# FIXME: This should boxplot_training(irace_results, type="ibest")
boxplot_performance(experiments = irace_results$experiments,
                    allElites = lapply(irace_results$allElites, utils::head, irace_results$scenario$testNbElites),
                    type = "ibest", interactive = interactive_plots)
```
```{r boxplot-train-iteration-elites-raw,fig.align="center", out.width="100%", echo=FALSE}
boxplot_performance(experiments = irace_results$experiments,
                    allElites = lapply(irace_results$allElites, utils::head, irace_results$scenario$testNbElites),
                    type = "ibest", rpd = FALSE, interactive = interactive_plots)
```

# Races overview

```{r experiments-matrix,fig.align="center", out.width="100%", echo=FALSE, eval=isTRUE(sections$experiments_matrix)}
plot_experiments_matrix(irace_results, interactive = interactive_plots)
```

```{asis, echo=!isTRUE(sections$experiments_matrix)}
Disabled because `sections$experiments_matrix` is FALSE.
```

# Convergence plot

This is a simplified version of the visualization you can obtain with [acviz](https://github.com/souzamarcelo/acviz).


```{r configurations-display,fig.align="center", out.width="100%", echo=FALSE, eval=isTRUE(sections$convergence)}
configurations_display(irace_results, interactive = FALSE)
```

```{asis, echo=!isTRUE(sections$convergence)}
Disabled because `sections$convergence` is FALSE.
```

---

Report generated by [iraceplot](https://auto-optimization.github.io/iraceplot/) version `r packageVersion("iraceplot")`.
