# Comparing cross-validation results using crossval_ml and boxplots

[This article was first published on

Want to share your content on R-bloggers? click here if you have a blog, or here if you don't.

**T. Moudiki's Webpage - R**, and kindly contributed to R-bloggers]. (You can report issue about the content on this page here)Want to share your content on R-bloggers? click here if you have a blog, or here if you don't.

# Table of contents

- 0 – Install packages + global parameters
- 1 – Regression example
- 2 – Classification example

# 0 – Install packages + global parameters

Let’s start by installing the main package, `crossvalidation`

(version 0.5.0):

**1st method**: from R-universe (where you can also package’s long-form descriptions a.k.a vignettes)

In R console:

options(repos = c( techtonique = 'https://techtonique.r-universe.dev', CRAN = 'https://cloud.r-project.org')) install.packages("crossvalidation")

**2nd method**: from Github

In R console:

remotes::install_github("Techtonique/crossvalidation")

When using this package, please note that I’m calling a “validation set”, what is usually called a “test set”. Because it makes more sense to me (even if I’m the only one in the world doing this).

Number of folds and repeats for the cross-validation procedure:

(n_folds <- 10) (repeats <- 5)

Loading the other Statistical/Machine Learning packages needed for this post:

library(glmnet) library(xgboost) library(Matrix) library(randomForest) library(crossvalidation)

# 1 - Regression example

# dataset set.seed(123) n <- 100 ; p <- 5 X <- matrix(rnorm(n * p), n, p) print(head(X)) y <- rnorm(n) print(head(y))

## least squares

# linear model example (cv_lm <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE))

`glmnet`

# glmnet example ----- # fit glmnet, with alpha = 1, lambda = 0.1 (cv_glmnet <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet, predict_func = predict, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01)))

## Random Forest

# randomForest example ----- # fit randomForest with mtry = 4 ( cv_rf <- crossvalidation::crossval_ml( x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = randomForest::randomForest, predict_func = predict, packages = "randomForest", fit_params = list(mtry = 4) ) )

`xgboost`

# xgboost example ----- # The response and covariates are named 'label' and 'data' # So, we do this: f_xgboost <- function(x, y, ...) xgboost::xgboost(data = x, label = y, ...) # fit xgboost with nrounds = 10 ( cv_xgboost <- crossvalidation::crossval_ml( x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = f_xgboost, predict_func = predict, #packages = "xgboost", fit_params = list(nrounds = 10, verbose = FALSE) ) )

`glmnet`

# glmnet example ----- # fit glmnet, with alpha = 0.5, lambda = 0.1 cv_glmnet1 <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet, predict_func = predict.glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0.5, lambda = 0.1, family = "gaussian")) # fit glmnet, with alpha = 0, lambda = 0.01 cv_glmnet2 <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet::glmnet, predict_func = predict.glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01, family = "gaussian")) # fit glmnet, with alpha = 0, lambda = 0.01 cv_glmnet3 <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet::glmnet, predict_func = predict.glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01))

## boxplots for regression

(samples <- crossvalidation::create_samples(cv_lm, cv_glmnet1, cv_glmnet2, cv_glmnet3, cv_rf, cv_xgboost, model_names = c("lm", "glmnet1", "glmnet2", "glmnet3", "rf", "xgb"))) boxplot(samples, main = "RMSE")

# 2 - Classification example

data(iris) X <- as.matrix(iris[, 1:4]) print(head(X)) y <- factor(as.numeric(iris$Species)) print(head(y))

`glmnet`

# glmnet example ----- predict_glmnet <- function(object, newx) { as.numeric(predict(object = object, newx = newx, type = "class")) } (cv_glmnet_1 <- crossvalidation::crossval_ml(x = X, y = as.integer(iris$Species), k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet, predict_func = predict_glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0.5, lambda = 0.1, family = "multinomial"))) # better to use `nlambda` (cv_glmnet_2 <- crossvalidation::crossval_ml(x = X, y = as.integer(iris$Species), k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet::glmnet, predict_func = predict_glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01, family = "multinomial"))) (cv_glmnet_3 <- crossvalidation::crossval_ml(x = X, y = as.integer(iris$Species) , k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = glmnet::glmnet, predict_func = predict_glmnet, packages = c("glmnet", "Matrix"), fit_params = list(alpha = 1, lambda = 0.01, family = "multinomial")))

## Random Forest

# randomForest example ----- # fit randomForest with mtry = 4 ( cv_rf <- crossvalidation::crossval_ml( x = X, y = y, k = n_folds, repeats = repeats, show_progress = FALSE, fit_func = randomForest::randomForest, predict_func = predict, #packages = "randomForest", fit_params = list(mtry = 2L) ) )

## xgboost

y <- as.integer(iris$Species) - 1 print(y) # xgboost example ----- # fit xgboost with nrounds = 10 f_xgboost <- function(x, y, ...) { #xgb_train = xgb.DMatrix(data=x, label=y) xgboost::xgboost(data = x, label = y, ...) } (cv_xgboost <- crossvalidation::crossval_ml(x = X, y = y, k = n_folds, repeats = repeats, fit_func = f_xgboost, predict_func = predict, packages = "xgboost", show_progress = FALSE, fit_params = list(nrounds = 50L, verbose = FALSE, params = list(max_depth = 3L, eta = 0.1, subsample = 0.8, colsample_bytree = 0.8, objective = "multi:softmax", num_class = 3L))))

## boxplots for classification

(samples <- crossvalidation::create_samples(cv_rf, cv_glmnet_1, cv_glmnet_2, cv_glmnet_3, cv_xgboost, model_names = c("rf", "glmnet1", "glmnet2", "glmnet3", "xgb"))) boxplot(samples, main = "Accuracy") abline(h = 1, col = "red", lty = 2, lwd = 2)

To

**leave a comment**for the author, please follow the link and comment on their blog:**T. Moudiki's Webpage - R**.R-bloggers.com offers

**daily e-mail updates**about R news and tutorials about learning R and many other topics. Click here if you're looking to post or find an R/data-science job.

Want to share your content on R-bloggers? click here if you have a blog, or here if you don't.