# Introduction to roclab

    library(roclab)

# Overview

The roclab package implements ROC-Optimizing Binary Classifiers, supporting both linear and kernel models.

- Linear models provide various regularization penalties (ridge, lasso, alasso, elastic net, scad, mcp).
- Kernel models support flexible kernel functions (radial, polynomial, linear, laplace).
- Both models include several surrogate loss functions (hinge, hinge2 (squared hinge), logistic, exponential).

For large-scale data, the model is computationally prohibitive because its loss is a U-statistic involving a double summation. To reduce this burden, the package adopts an efficient algorithm based on an incomplete U-statistic, which approximates the loss with a single summation. In kernel models, a Nyström low-rank approximation is further applied to efficiently compute the kernel matrix. These approximations substantially reduce computational cost and accelerate training, while maintaining accuracy, making ROC-Optimizing Binary Classifiers practical for large-scale datasets.

In addition, efficient optimization is performed using Gradient Descent with the Adamax update rule —a variant of Adam, based on the infinity norm—for linear models with the ridge penalty and kernel models. For linear models with other penalties (i.e., those involving variable selection), Proximal Gradient Descent with an Adamax adaptive learning rate scheme is employed.

# Example: Linear Model

    set.seed(123)
    n_lin_train <- 1200
    n_pos_lin <- round(0.2 * n_lin_train )
    n_neg_lin <- n_lin_train  - n_pos_lin

    X_train_lin <- rbind(
      matrix(rnorm(2 * n_neg_lin, mean = -1), ncol = 2),
      matrix(rnorm(2 * n_pos_lin, mean =  1), ncol = 2)
    )
    y_train_lin <- c(rep(-1, n_neg_lin), rep(1, n_pos_lin))

    # Fit a linear model
    fit_lin <- roclearn(X_train_lin, y_train_lin, lambda = 0.1)

    # Summary
    summary(fit_lin)
    #> Linear Model Summary
    #> ----------------------
    #> Call:
    #> roclearn(X = X_train_lin, y = y_train_lin, lambda = 0.1)
    #>
    #> Number of observations: 1200
    #> Number of predictors: 2
    #>
    #> Penalty: ridge
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (B = 1200 )
    #>
    #> Intercept: -0.0678
    #> Converged: TRUE
    #> Iterations: 1179
    #> Training time (sec): 0.05
    #>
    #> Coefficients (first 10):
    #>        V1        V2
    #> 0.4726011 0.4684517

    n_test_lin <- 300
    n_pos_test_lin <- round(0.2 * n_test_lin)
    n_neg_test_lin <- n_test_lin - n_pos_test_lin
    X_test_lin <- rbind(
      matrix(rnorm(2 * n_neg_test_lin, mean = -1), ncol = 2),
      matrix(rnorm(2 * n_pos_test_lin, mean =  1), ncol = 2)
    )
    y_test_lin <- c(rep(-1, n_neg_test_lin), rep(1, n_pos_test_lin))

    # Predict decision scores on the test set
    pred_score_lin <- predict(fit_lin, X_test_lin, type = "response")
    head(pred_score_lin)
    #> [1] -1.7635218 -1.3635854 -0.8779351 -1.4472523 -0.1029838 -1.2500200

    # Predict classes {-1, 1} on the test set
    pred_class_lin <- predict(fit_lin, X_test_lin, type = "class")
    head(pred_class_lin)
    #> [1] -1 -1 -1 -1 -1 -1

    # AUC on the test set
    auc(fit_lin, X_test_lin, y_test_lin)
    #> [1] 0.9763194

    # Plot ROC curve on the test set
    plot_roc(y_test_lin, pred_score_lin)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases
    #> Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
    #> ℹ Please use `linewidth` instead.
    #> ℹ The deprecated feature was likely used in the pROC package.
    #>   Please report the issue at <https://github.com/xrobin/pROC/issues>.
    #> This warning is displayed once every 8 hours.
    #> Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
    #> generated.

# Example: Kernel Model

    set.seed(123)
    n_ker <- 1200
    r_train_ker <- sqrt(runif(n_ker, 0.05, 1))
    theta_train_ker <- runif(n_ker, 0, 2*pi)
    X_train_ker <- cbind(r_train_ker * cos(theta_train_ker), r_train_ker * sin(theta_train_ker))
    y_train_ker <- ifelse(r_train_ker < 0.5, 1, -1)

    # Fit a kernel model
    fit_ker <- kroclearn(X_train_ker, y_train_ker, lambda = 0.1, kernel = "radial")

    # Summary
    summary(fit_ker)
    #> Kernel Model Summary
    #> -----------------------------
    #> Call:
    #> kroclearn(X = X_train_ker, y = y_train_ker, lambda = 0.1, kernel = "radial")
    #>
    #> Number of observations: 1200
    #> Number of predictors: 2
    #>
    #> Kernel: radial
    #> Kernel parameter: 0.5
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (Nystrom, B = 1200 )
    #>   Nystrom landmarks: 35
    #>
    #> Intercept: -1.4187
    #> Converged: TRUE
    #> Iterations: 1117
    #> Training time (sec): 0.5
    #>
    #> Theta coefficients (first 10):
    #>  [1]  0.07866332 -0.03783574  0.10868490  0.37010068  0.09839754 -0.03419196
    #>  [7]  0.34827559  0.46628109  0.15124239  1.07986830
    #> ...

    n_test_ker <- 300
    r_test_ker <- sqrt(runif(n_test_ker, 0.05, 1))
    theta_test_ker <- runif(n_test_ker, 0, 2*pi)
    X_test_ker <- cbind(r_test_ker * cos(theta_test_ker), r_test_ker * sin(theta_test_ker))
    y_test_ker <- ifelse(r_test_ker < 0.5, 1, -1)

    # Predict decision scores on the test set
    pred_score_ker <- predict(fit_ker, X_test_ker, type = "response")
    head(pred_score_ker)
    #> [1]  0.28910462 -0.07075436 -0.17158776 -0.59294103  0.06180517 -0.12245284

    # Predict classes {-1, 1} on the test set
    pred_class_ker <- predict(fit_ker, X_test_ker, type = "class")
    head(pred_class_ker)
    #> [1]  1 -1 -1 -1  1 -1

    # AUC on the test set
    auc(fit_ker, X_test_ker, y_test_ker)
    #> [1] 0.9942396

    # Plot ROC curve on the test set
    plot_roc(y_test_ker, pred_score_ker)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases

# Cross-Validation

    # 5-fold CV for linear models
    cvfit_lin <- cv.roclearn(
      X_train_lin, y_train_lin,
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 20)),
      nfolds = 5
    )

    # Summarize the cross-validation result
    summary(cvfit_lin)
    #> Cross-validated Linear Model
    #> -----------------------
    #> Loss: hinge
    #> Penalty: ridge
    #> Number of folds: 5
    #> Number of candidate lambdas: 20
    #>
    #> Optimal lambda: 0.05131629
    #> Cross-validated AUC at optimal lambda: 0.9694 (+/- 0.0103)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.9687 0.0099
    #> 2  0.0139   0.9687 0.0096
    #> 3  0.0192   0.9688 0.0095
    #> 4  0.0267   0.9690 0.0104
    #> 5  0.0370   0.9690 0.0100
    #> 6  0.0513   0.9694 0.0103
    #> 7  0.0712   0.9688 0.0106
    #> 8  0.0987   0.9692 0.0104
    #> 9  0.1370   0.9692 0.0102
    #> 10 0.1900   0.9692 0.0103
    #> ...

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_lin)

    # 5-fold CV for kernel models
    cvfit_ker <- cv.kroclearn(
      X_train_ker, y_train_ker,
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 20)),
      kernel = "radial",
      nfolds = 5
    )

    # Summarize the cross-validation result
    summary(cvfit_ker)
    #> Cross-validated Kernel Model
    #> ------------------------------
    #> Loss: hinge
    #> Kernel: radial
    #> Number of folds: 5
    #> Number of candidate lambdas: 20
    #>
    #> Optimal lambda: 0.01386919
    #> Cross-validated AUC at optimal lambda: 0.9994 (+/- 0.0005)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.9981 0.0031
    #> 2  0.0139   0.9994 0.0005
    #> 3  0.0192   0.9984 0.0025
    #> 4  0.0267   0.9992 0.0012
    #> 5  0.0370   0.9986 0.0018
    #> 6  0.0513   0.9987 0.0016
    #> 7  0.0712   0.9972 0.0017
    #> 8  0.0987   0.9947 0.0047
    #> 9  0.1370   0.9868 0.0136
    #> 10 0.1900   0.9833 0.0076
    #> ...

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_ker)

# Real Data Example 1: Ionosphere

    library(mlbench)
    #> Warning: package 'mlbench' was built under R version 4.4.3
    data(Ionosphere)

    # Prepare data
    X_iono <- Ionosphere[, -35]
    y_iono <- ifelse(Ionosphere$Class == "bad", 1, -1)

    set.seed(123)
    train_idx <- sample(seq_len(nrow(X_iono)), size = 200)
    X_train_iono <- X_iono[train_idx, ]
    y_train_iono <- y_iono[train_idx]
    X_test_iono  <- X_iono[-train_idx, ]
    y_test_iono  <- y_iono[-train_idx]

    # Fit a linear model
    fit_iono_lin <- roclearn(X_train_iono, y_train_iono, lambda = 0.1, approx=TRUE)
    #> Warning: Removing constant columns: V2
    #> [V1]: treated as categorical and one-hot encoded.
    summary(fit_iono_lin)
    #> Linear Model Summary
    #> ----------------------
    #> Call:
    #> roclearn(X = X_train_iono, y = y_train_iono, lambda = 0.1, approx = TRUE)
    #>
    #> Number of observations: 200
    #> Number of predictors: 33
    #>
    #> Penalty: ridge
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (B = 200 )
    #>
    #> Intercept: 1.3041
    #> Converged: TRUE
    #> Iterations: 1164
    #> Training time (sec): 0.07
    #>
    #> Coefficients (first 10):
    #>          V3          V4          V5          V6          V7          V8
    #> -0.40674510 -0.18973373 -0.42204641 -0.22075386 -0.17022688 -0.43640505
    #>          V9         V10         V11         V12
    #> -0.12857765 -0.24904508  0.08956809 -0.11456464
    #> ...
    #>
    #> Categorical variables dummy-encoded:
    #> [1] "V1"
    #> Removed columns:
    #> [1] "V2"

    # Predict decision scores on the test set
    pred_score_iono_lin <- predict(fit_iono_lin, X_test_iono, type = "response")
    head(pred_score_iono_lin)
    #> [1]  0.7813989 -0.4333132  1.0361641  1.5973041  0.2972169 -0.4974081

    # Predict classes {-1, 1} on the test set
    pred_class_iono_lin <- predict(fit_iono_lin, X_test_iono, type = "class")
    head(pred_class_iono_lin)
    #> [1]  1 -1  1  1  1 -1

    # AUC on the test set
    auc(fit_iono_lin, X_test_iono, y_test_iono)
    #> [1] 0.9184803

    # Plot ROC curve on the test set
    plot_roc(y_test_iono, pred_score_iono_lin)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases

    # 5-fold CV for linear models
    cvfit_iono_lin <- cv.roclearn(
      X_train_iono, y_train_iono,
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 10)),
      approx=TRUE, nfolds=5)
    #> Warning: Removing constant columns: V2
    #> [V1]: treated as categorical and one-hot encoded.
    summary(cvfit_iono_lin)
    #> Cross-validated Linear Model
    #> -----------------------
    #> Loss: hinge
    #> Penalty: ridge
    #> Number of folds: 5
    #> Number of candidate lambdas: 10
    #>
    #> Optimal lambda: 0.3158114
    #> Cross-validated AUC at optimal lambda: 0.8633 (+/- 0.0965)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.8072 0.1232
    #> 2  0.0199   0.8236 0.1222
    #> 3  0.0398   0.8408 0.0943
    #> 4  0.0794   0.8624 0.0794
    #> 5  0.1580   0.8501 0.0866
    #> 6  0.3160   0.8633 0.0965
    #> 7  0.6300   0.8489 0.0896
    #> 8  1.2600   0.8250 0.1270
    #> 9  2.5100   0.8068 0.1361
    #> 10 5.0000   0.7776 0.1213

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_iono_lin)

    # Fit a kernel model
    fit_iono_ker <- kroclearn(X_train_iono, y_train_iono, lambda = 0.1, kernel = "radial", approx=TRUE)
    #> Warning: Removing constant columns: V2
    #> [V1]: treated as categorical and one-hot encoded.
    summary(fit_iono_ker)
    #> Kernel Model Summary
    #> -----------------------------
    #> Call:
    #> kroclearn(X = X_train_iono, y = y_train_iono, lambda = 0.1, kernel = "radial",
    #>     approx = TRUE)
    #>
    #> Number of observations: 200
    #> Number of predictors: 33
    #>
    #> Kernel: radial
    #> Kernel parameter: 0.03030303
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (Nystrom, B = 200 )
    #>   Nystrom landmarks: 14
    #>
    #> Intercept: 0.7425
    #> Converged: TRUE
    #> Iterations: 1276
    #> Training time (sec): 0.05
    #>
    #> Theta coefficients (first 10):
    #>  [1] -0.87955477  0.22849258 -0.33857814 -0.67966979 -0.04122377 -0.17061431
    #>  [7]  0.08362898 -0.76337429  0.35662077 -0.52693062
    #> ...
    #>
    #> Categorical variables dummy-encoded:
    #> [1] "V1"
    #> Removed columns:
    #> [1] "V2"

    # Predict decision scores on the test set
    pred_score_iono_ker <- predict(fit_iono_ker, X_test_iono, type = "response")
    head(pred_score_iono_ker)
    #> [1]  0.1672607 -0.6231523  0.2522829  0.4332831 -0.2635053 -0.6066902

    # Predict classes {-1, 1} on the test set
    pred_class_iono_ker <- predict(fit_iono_ker, X_test_iono, type = "class")
    head(pred_class_iono_ker)
    #> [1]  1 -1  1  1 -1 -1

    # AUC on the test set
    auc(fit_iono_ker, X_test_iono, y_test_iono)
    #> [1] 0.8999618

    # Plot ROC curve on the test set
    plot_roc(y_test_iono, pred_score_iono_ker)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases

    # 5-fold CV for kernel models
    cvfit_iono_ker <- cv.kroclearn(
      X_train_iono, y_train_iono,
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 10)),
      kernel = "radial",
      approx=TRUE, nfolds=5)
    #> Warning: Removing constant columns: V2
    #> [V1]: treated as categorical and one-hot encoded.
    summary(cvfit_iono_ker)
    #> Cross-validated Kernel Model
    #> ------------------------------
    #> Loss: hinge
    #> Kernel: radial
    #> Number of folds: 5
    #> Number of candidate lambdas: 10
    #>
    #> Optimal lambda: 0.01994737
    #> Cross-validated AUC at optimal lambda: 0.9153 (+/- 0.0226)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.8981 0.0560
    #> 2  0.0199   0.9153 0.0226
    #> 3  0.0398   0.8671 0.0773
    #> 4  0.0794   0.8608 0.0530
    #> 5  0.1580   0.8189 0.0580
    #> 6  0.3160   0.8189 0.0665
    #> 7  0.6300   0.7965 0.0450
    #> 8  1.2600   0.7944 0.0879
    #> 9  2.5100   0.8041 0.0649
    #> 10 5.0000   0.8137 0.0387

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_iono_ker)

# Real Data Example 2: Spam (larger dataset)

    library(kernlab)
    data(spam)

    # Prepare data
    X_spam <- spam[, -58]
    y_spam <- ifelse(spam$type == "spam", 1, -1)

    set.seed(123)
    train_idx <- sample(seq_len(nrow(X_spam)), size = 3000)
    X_train_spam <- X_spam[train_idx, ]
    y_train_spam <- y_spam[train_idx]
    X_test_spam  <- X_spam[-train_idx, ]
    y_test_spam  <- y_spam[-train_idx]

    # Fit a linear model
    fit_spam_lin <- roclearn(X_train_spam, y_train_spam, lambda = 0.1)
    summary(fit_spam_lin)
    #> Linear Model Summary
    #> ----------------------
    #> Call:
    #> roclearn(X = X_train_spam, y = y_train_spam, lambda = 0.1)
    #>
    #> Number of observations: 3000
    #> Number of predictors: 57
    #>
    #> Penalty: ridge
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (B = 3000 )
    #>
    #> Intercept: -0.5147
    #> Converged: TRUE
    #> Iterations: 10
    #> Training time (sec): 0.02
    #>
    #> Coefficients (first 10):
    #>          make       address           all         num3d           our
    #>  4.321339e-06 -5.145267e-06  1.017238e-05  5.731498e-06  1.621501e-05
    #>          over        remove      internet         order          mail
    #>  5.582747e-06  1.344279e-05  7.710516e-06  3.596203e-06  7.202070e-06
    #> ...

    # Predict decision scores on the test set
    pred_score_spam_lin <- predict(fit_spam_lin, X_test_spam, type = "response")
    head(pred_score_spam_lin)
    #>          3          6          8         10         14         15
    #> 10.4085857 -0.2464476 -0.2763554  2.8799679 -0.3900616  0.4386678

    # Predict classes {-1, 1} on the test set
    pred_class_spam_lin <- predict(fit_spam_lin, X_test_spam, type = "class")
    head(pred_class_spam_lin)
    #>  3  6  8 10 14 15
    #>  1 -1 -1  1 -1  1

    # AUC on the test set
    auc(fit_spam_lin, X_test_spam, y_test_spam)
    #> [1] 0.7624179

    # Plot ROC curve on the test set
    plot_roc(y_test_spam, pred_score_spam_lin)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases

    # 5-fold CV for linear models
    cvfit_spam_lin <- cv.roclearn(
      X_train_spam, y_train_spam,
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 10)), nfolds=5)
    summary(cvfit_spam_lin)
    #> Cross-validated Linear Model
    #> -----------------------
    #> Loss: hinge
    #> Penalty: ridge
    #> Number of folds: 5
    #> Number of candidate lambdas: 10
    #>
    #> Optimal lambda: 0.3158114
    #> Cross-validated AUC at optimal lambda: 0.7777 (+/- 0.0232)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.7715 0.0167
    #> 2  0.0199   0.7764 0.0194
    #> 3  0.0398   0.7741 0.0215
    #> 4  0.0794   0.7748 0.0220
    #> 5  0.1580   0.7758 0.0252
    #> 6  0.3160   0.7777 0.0232
    #> 7  0.6300   0.7735 0.0233
    #> 8  1.2600   0.7753 0.0192
    #> 9  2.5100   0.7715 0.0199
    #> 10 5.0000   0.7722 0.0224

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_spam_lin)

    # Fit a kernel model
    fit_spam_ker <- kroclearn(X_train_spam, y_train_spam, lambda = 0.1, kernel = "radial")
    summary(fit_spam_ker)
    #> Kernel Model Summary
    #> -----------------------------
    #> Call:
    #> kroclearn(X = X_train_spam, y = y_train_spam, lambda = 0.1, kernel = "radial")
    #>
    #> Number of observations: 3000
    #> Number of predictors: 57
    #>
    #> Kernel: radial
    #> Kernel parameter: 0.01754386
    #> Lambda: 0.1
    #> Loss function: hinge
    #> Approximation: Yes (Nystrom, B = 3000 )
    #>   Nystrom landmarks: 55
    #>
    #> Intercept: 5e-04
    #> Converged: TRUE
    #> Iterations: 607
    #> Training time (sec): 0.74
    #>
    #> Theta coefficients (first 10):
    #>  [1] -0.106694761  0.034588410 -0.155193553 -0.091769201  0.008358426
    #>  [6]  0.039672217 -0.191470994 -0.079506402  0.013024674 -0.468039804
    #> ...

    # Predict decision scores on the test set
    pred_score_spam_ker <- predict(fit_spam_ker, X_test_spam, type = "response")
    head(pred_score_spam_ker)
    #>             3             6             8            10            14
    #>  0.0005386429 -0.0461043933 -0.1537764925  0.0005388521 -0.3817359714
    #>            15
    #>  0.0005386429

    # Predict classes {-1, 1} on the test set
    pred_class_spam_ker <- predict(fit_spam_ker, X_test_spam, type = "class")
    head(pred_class_spam_ker)
    #>  3  6  8 10 14 15
    #>  1 -1 -1  1 -1  1

    # AUC on the test set
    auc(fit_spam_ker, X_test_spam, y_test_spam)
    #> [1] 0.7513072

    # Plot ROC curve on the test set
    plot_roc(y_test_spam, pred_score_spam_ker)
    #> Setting levels: control = -1, case = 1
    #> Setting direction: controls < cases

    # 5-fold CV for kernel models
    cvfit_spam_ker <- cv.kroclearn(
      X_train_spam, y_train_spam,
      kernel = "radial",
      lambda.vec = exp(seq(log(0.01), log(5), length.out = 10)), nfolds=5)
    summary(cvfit_spam_ker)
    #> Cross-validated Kernel Model
    #> ------------------------------
    #> Loss: hinge
    #> Kernel: radial
    #> Number of folds: 5
    #> Number of candidate lambdas: 10
    #>
    #> Optimal lambda: 5
    #> Cross-validated AUC at optimal lambda: 0.7385 (+/- 0.0146)
    #>
    #> Overall AUC (mean +/- sd) across lambdas:
    #>    lambda auc.mean auc.sd
    #> 1  0.0100   0.7303 0.0146
    #> 2  0.0199   0.7173 0.0339
    #> 3  0.0398   0.7301 0.0139
    #> 4  0.0794   0.7287 0.0167
    #> 5  0.1580   0.7258 0.0184
    #> 6  0.3160   0.7196 0.0162
    #> 7  0.6300   0.7281 0.0295
    #> 8  1.2600   0.7331 0.0168
    #> 9  2.5100   0.7259 0.0292
    #> 10 5.0000   0.7385 0.0146

    # Plot the cross-validation AUC across lambda values
    plot(cvfit_spam_ker)

# Conclusion

The roclab package implements ROC-Optimizing Binary Classifiers through both linear models (with regularization penalties) and kernel models (with various kernel functions). It supports multiple surrogate loss functions and incorporates scalable options to efficiently handle large datasets.