modeltime_fit_resamples does not work with ensemble

[vidarsumo]

This code will run but the ensemble is not present when I calculate accuracy:

# Create average ensemble and add to the modeltime table
ml_mtbl <- ml_mtbl %>% 
    combine_modeltime_tables(
        ml_mtbl %>% 
            ensemble_average() %>% 
            modeltime_table()
    )

# TS CV
resamples_tscv <- time_series_cv(
    data        = train_data,
    assess      = "11 days",
    initial     = "730 days",
    skip        = 11,
    slice_limit = 20,
    cumulative = TRUE
    )

resamples_fitted <- ml_mtbl %>% 
    modeltime_fit_resamples(
        resamples = resamples_tscv,
        control   = control_resamples(verbose = FALSE, allow_par = TRUE)
    )

resamples_fitted
# Modeltime Table
# A tibble: 6 x 4
  .model_id .model         .model_desc               .resample_results
      <int> <list>         <chr>                     <list>           
1         1 <workflow>     XGBOOST                   <rsmp[+]>        
2         2 <workflow>     RANGER                    <rsmp[+]>        
3         3 <workflow>     GLMNET                    <rsmp[+]>        
4         4 <workflow>     KERNLAB                   <rsmp[+]>        
5         5 <workflow>     KERNLAB                   <rsmp[+]>        
6         6 <ensemble [5]> ENSEMBLE (MEAN): 5 MODELS <lgl [1]>      

resamples_fitted %>%
    modeltime_resample_accuracy()

I.e. the output of modeltime_resample_accuracy() will only include the models, not the ensemble.

Same if I have a table with stacked ensembles and try to run this:

model_stack_level_2_mtbl
# Modeltime Table
# A tibble: 5 x 3
  .model_id .model         .model_desc                       
      <int> <list>         <chr>                             
1         1 <ensemble [9]> ENSEMBLE (GLMNET STACK): 9 MODELS 
2         2 <ensemble [9]> ENSEMBLE (RANGER STACK): 9 MODELS 
3         3 <ensemble [9]> ENSEMBLE (XGBOOST STACK): 9 MODELS
4         4 <ensemble [9]> ENSEMBLE (CUBIST STACK): 9 MODELS 
5         5 <ensemble [9]> ENSEMBLE (KERNLAB STACK): 9 MODELS

stacking_rsample_tscv <- model_stack_level_2_mtbl %>%
    modeltime_fit_resamples(
        resamples = test_data,
        control = control_resamples(
            verbose = TRUE,
            pkgs = c("catboost", "treesnip", "Cubist", "rules")
        )
    )

-- Fitting Resamples --------------------------------------------

* Model ID: 1 ENSEMBLE (GLMNET STACK): 9 MODELS
Error: no applicable method for 'mdl_time_fit_resamples' applied to an object of class "c('mdl_time_ensemble_model_spec', 'mdl_time_ensemble')"
* Model ID: 2 ENSEMBLE (RANGER STACK): 9 MODELS
Error: no applicable method for 'mdl_time_fit_resamples' applied to an object of class "c('mdl_time_ensemble_model_spec', 'mdl_time_ensemble')"
* Model ID: 3 ENSEMBLE (XGBOOST STACK): 9 MODELS
Error: no applicable method for 'mdl_time_fit_resamples' applied to an object of class "c('mdl_time_ensemble_model_spec', 'mdl_time_ensemble')"
* Model ID: 4 ENSEMBLE (CUBIST STACK): 9 MODELS
Error: no applicable method for 'mdl_time_fit_resamples' applied to an object of class "c('mdl_time_ensemble_model_spec', 'mdl_time_ensemble')"
* Model ID: 5 ENSEMBLE (KERNLAB STACK): 9 MODELS
Error: no applicable method for 'mdl_time_fit_resamples' applied to an object of class "c('mdl_time_ensemble_model_spec', 'mdl_time_ensemble')"

Session info:

- Session info ------------------------------------------------------------------------------------------------------------------------------------------------------------------
 setting  value                       
 version  R version 4.1.0 (2021-05-18)
 os       Windows 10 x64              
 system   x86_64, mingw32             
 ui       RStudio                     
 language (EN)                        
 collate  Icelandic_Iceland.1252      
 ctype    Icelandic_Iceland.1252      
 tz       Africa/Casablanca           
 date     2021-08-16                  

- Packages ----------------------------------------------------------------------------------------------------------------------------------------------------------------------
 ! package            * version    date       lib source                                              
   askpass              1.1        2019-01-13 [1] CRAN (R 4.1.0)                                      
   assertthat           0.2.1      2019-03-21 [1] CRAN (R 4.1.0)                                      
   AzureAuth            1.3.2      2021-05-19 [1] CRAN (R 4.1.0)                                      
   AzureGraph           1.3.1      2021-06-04 [1] CRAN (R 4.1.0)                                      
   AzureKeyVault      * 1.0.4      2020-10-12 [1] CRAN (R 4.1.0)                                      
   AzureRMR             2.4.2      2021-06-03 [1] CRAN (R 4.1.0)                                      
   AzureStor          * 3.5.0      2021-08-03 [1] Github (Azure/AzureStor@19f879b)                    
   backports            1.2.1      2020-12-09 [1] CRAN (R 4.1.0)                                      
   broom              * 0.7.9      2021-07-27 [1] CRAN (R 4.1.0)                                      
   catboost             0.25.1     2021-05-28 [1] url                                                 
   cellranger           1.1.0      2016-07-27 [1] CRAN (R 4.1.0)                                      
   class                7.3-19     2021-05-03 [2] CRAN (R 4.1.0)                                      
   cli                  3.0.1      2021-07-17 [1] CRAN (R 4.1.0)                                      
   codetools            0.2-18     2020-11-04 [2] CRAN (R 4.1.0)                                      
   colorspace           2.0-2      2021-06-24 [1] CRAN (R 4.1.0)                                      
   crayon               1.4.1      2021-02-08 [1] CRAN (R 4.1.0)                                      
   crosstalk            1.1.1      2021-01-12 [1] CRAN (R 4.1.0)                                      
   Cubist               0.3.0      2021-05-28 [1] CRAN (R 4.1.0)                                      
   curl                 4.3.2      2021-06-23 [1] CRAN (R 4.1.0)                                      
   data.table           1.14.0     2021-02-21 [1] CRAN (R 4.1.0)                                      
   DBI                  1.1.1      2021-01-15 [1] CRAN (R 4.1.0)                                      
   dbplyr               2.1.1      2021-04-06 [1] CRAN (R 4.1.0)                                      
   dials              * 0.0.9.9000 2021-07-29 [1] Github (tidymodels/dials@dc9e020)                   
   DiceDesign           1.9        2021-02-13 [1] CRAN (R 4.1.0)                                      
   digest               0.6.27     2020-10-24 [1] CRAN (R 4.1.0)                                      
   doFuture           * 0.12.0     2021-01-04 [1] CRAN (R 4.1.0)                                      
   dplyr              * 1.0.7      2021-06-18 [1] CRAN (R 4.1.0)                                      
   ellipsis             0.3.2      2021-04-29 [1] CRAN (R 4.1.0)                                      
   fansi                0.5.0      2021-05-25 [1] CRAN (R 4.1.0)                                      
   finetune           * 0.1.0      2021-07-21 [1] CRAN (R 4.1.0)                                      
   forcats            * 0.5.1      2021-01-27 [1] CRAN (R 4.1.0)                                      
   foreach            * 1.5.1      2020-10-15 [1] CRAN (R 4.1.0)                                      
   fs                   1.5.0      2020-07-31 [1] CRAN (R 4.1.0)                                      
   furrr                0.2.3      2021-06-25 [1] CRAN (R 4.1.0)                                      
   future             * 1.21.0     2020-12-10 [1] CRAN (R 4.1.0)                                      
   generics             0.1.0      2020-10-31 [1] CRAN (R 4.1.0)                                      
   ggplot2            * 3.3.5      2021-06-25 [1] CRAN (R 4.1.0)                                      
   glmnet               4.1-2      2021-06-24 [1] CRAN (R 4.1.0)                                      
   globals              0.14.0     2020-11-22 [1] CRAN (R 4.1.0)                                      
   glue                 1.4.2      2020-08-27 [1] CRAN (R 4.1.0)                                      
   gower                0.2.2      2020-06-23 [1] CRAN (R 4.1.0)                                      
   GPfit                1.0-8      2019-02-08 [1] CRAN (R 4.1.0)                                      
   gridExtra            2.3        2017-09-09 [1] CRAN (R 4.1.0)                                      
   gtable               0.3.0      2019-03-25 [1] CRAN (R 4.1.0)                                      
   hardhat              0.1.6      2021-07-14 [1] CRAN (R 4.1.0)                                      
   haven                2.4.3      2021-08-04 [1] CRAN (R 4.1.0)                                      
   hms                  1.1.0      2021-05-17 [1] CRAN (R 4.1.0)                                      
   htmltools            0.5.1.1    2021-01-22 [1] CRAN (R 4.1.0)                                      
   htmlwidgets          1.5.3      2020-12-10 [1] CRAN (R 4.1.0)                                      
   httr                 1.4.2      2020-07-20 [1] CRAN (R 4.1.0)                                      
   infer              * 0.5.4      2021-01-13 [1] CRAN (R 4.1.0)                                      
   ipred                0.9-11     2021-03-12 [1] CRAN (R 4.1.0)                                      
   iterators            1.0.13     2020-10-15 [1] CRAN (R 4.1.0)                                      
   janitor              2.1.0      2021-01-05 [1] CRAN (R 4.1.0)                                      
   jsonlite             1.7.2      2020-12-09 [1] CRAN (R 4.1.0)                                      
   kernlab              0.9-29     2019-11-12 [1] CRAN (R 4.1.0)                                      
   labeling             0.4.2      2020-10-20 [1] CRAN (R 4.1.0)                                      
   lattice              0.20-44    2021-05-02 [2] CRAN (R 4.1.0)                                      
   lava                 1.6.9      2021-03-11 [1] CRAN (R 4.1.0)                                      
   lazyeval             0.2.2      2019-03-15 [1] CRAN (R 4.1.0)                                      
   lhs                  1.1.1      2020-10-05 [1] CRAN (R 4.1.0)                                      
   lifecycle            1.0.0      2021-02-15 [1] CRAN (R 4.1.0)                                      
   listenv              0.8.0      2019-12-05 [1] CRAN (R 4.1.0)                                      
   lubridate          * 1.7.10     2021-02-26 [1] CRAN (R 4.1.0)                                      
   magrittr             2.0.1      2020-11-17 [1] CRAN (R 4.1.0)                                      
   MASS                 7.3-54     2021-05-03 [2] CRAN (R 4.1.0)                                      
   Matrix               1.3-3      2021-05-04 [2] CRAN (R 4.1.0)                                      
   mime                 0.11       2021-06-23 [1] CRAN (R 4.1.0)                                      
   modeldata          * 0.1.1      2021-07-14 [1] CRAN (R 4.1.0)                                      
   modelr               0.1.8      2020-05-19 [1] CRAN (R 4.1.0)                                      
   modeltime          * 0.7.0      2021-07-16 [1] CRAN (R 4.1.0)                                      
   modeltime.ensemble * 0.4.2      2021-07-16 [1] CRAN (R 4.1.0)                                      
   modeltime.resample * 0.2.0      2021-05-27 [1] Github (business-science/modeltime.resample@abde5ac)
   munsell              0.5.0      2018-06-12 [1] CRAN (R 4.1.0)                                      
   nnet                 7.3-16     2021-05-03 [2] CRAN (R 4.1.0)                                      
   openssl              1.4.4      2021-04-30 [1] CRAN (R 4.1.0)                                      
   parallelly           1.27.0     2021-07-19 [1] CRAN (R 4.1.0)                                      
   parsnip            * 0.1.7      2021-07-21 [1] CRAN (R 4.1.0)                                      
   pillar               1.6.2      2021-07-29 [1] CRAN (R 4.1.0)                                      
   pkgconfig            2.0.3      2019-09-22 [1] CRAN (R 4.1.0)                                      
   plotly               4.9.4.1    2021-06-18 [1] CRAN (R 4.1.0)                                      
   plyr                 1.8.6      2020-03-03 [1] CRAN (R 4.1.0)                                      
   png                  0.1-7      2013-12-03 [1] CRAN (R 4.1.0)                                      
   pROC                 1.17.0.1   2021-01-13 [1] CRAN (R 4.1.0)                                      
   prodlim              2019.11.13 2019-11-17 [1] CRAN (R 4.1.0)                                      
   progressr            0.8.0      2021-06-10 [1] CRAN (R 4.1.0)                                      
   prophet              1.0        2021-03-30 [1] CRAN (R 4.1.0)                                      
   purrr              * 0.3.4      2020-04-17 [1] CRAN (R 4.1.0)                                      
   R6                   2.5.0      2020-10-28 [1] CRAN (R 4.1.0)                                      
   ranger               0.13.1     2021-07-14 [1] CRAN (R 4.1.0)                                      
   rappdirs             0.3.3      2021-01-31 [1] CRAN (R 4.1.0)                                      
   Rcpp                 1.0.7      2021-07-07 [1] CRAN (R 4.1.0)                                      
 D RcppParallel         5.1.4      2021-05-04 [1] CRAN (R 4.1.0)                                      
   readr              * 2.0.0      2021-07-20 [1] CRAN (R 4.1.0)                                      
   readxl               1.3.1      2019-03-13 [1] CRAN (R 4.1.0)                                      
   recipes            * 0.1.16     2021-04-16 [1] CRAN (R 4.1.0)                                      
   reprex               2.0.1      2021-08-05 [1] CRAN (R 4.1.0)                                      
   reshape2             1.4.4      2020-04-09 [1] CRAN (R 4.1.0)                                      
   reticulate           1.20       2021-05-03 [1] CRAN (R 4.1.0)                                      
   rlang              * 0.4.11     2021-04-30 [1] CRAN (R 4.1.0)                                      
   rpart                4.1-15     2019-04-12 [2] CRAN (R 4.1.0)                                      
   rsample            * 0.1.0      2021-05-08 [1] CRAN (R 4.1.0)                                      
   rstudioapi           0.13       2020-11-12 [1] CRAN (R 4.1.0)                                      
   rules              * 0.1.2      2021-08-07 [1] CRAN (R 4.1.0)                                      
   rvest                1.0.1      2021-07-26 [1] CRAN (R 4.1.0)                                      
   scales             * 1.1.1      2020-05-11 [1] CRAN (R 4.1.0)                                      
   sessioninfo          1.1.1      2018-11-05 [1] CRAN (R 4.1.0)                                      
   shape                1.4.6      2021-05-19 [1] CRAN (R 4.1.0)                                      
   snakecase            0.11.0     2019-05-25 [1] CRAN (R 4.1.0)                                      
   StanHeaders          2.21.0-7   2020-12-17 [1] CRAN (R 4.1.0)                                      
   stringi              1.7.3      2021-07-16 [1] CRAN (R 4.1.0)                                      
   stringr            * 1.4.0      2019-02-10 [1] CRAN (R 4.1.0)                                      
   sumots             * 0.1.0      2021-08-16 [1] local                                               
   survival             3.2-11     2021-04-26 [2] CRAN (R 4.1.0)                                      
   tibble             * 3.1.3      2021-07-23 [1] CRAN (R 4.1.0)                                      
   tictoc             * 1.0.1      2021-04-19 [1] CRAN (R 4.1.0)                                      
   tidymodels         * 0.1.3.9000 2021-05-27 [1] Github (tidymodels/tidymodels@69ebce8)              
   tidyr              * 1.1.3      2021-03-03 [1] CRAN (R 4.1.0)                                      
   tidyselect           1.1.1      2021-04-30 [1] CRAN (R 4.1.0)                                      
   tidyverse          * 1.3.1      2021-04-15 [1] CRAN (R 4.1.0)                                      
   timeDate             3043.102   2018-02-21 [1] CRAN (R 4.1.0)                                      
   timetk             * 2.6.1      2021-01-18 [1] CRAN (R 4.1.0)                                      
   treesnip           * 0.1.0.9000 2021-05-31 [1] Github (curso-r/treesnip@bf27cd8)                   
   tune               * 0.1.6      2021-07-21 [1] CRAN (R 4.1.0)                                      
   tzdb                 0.1.2      2021-07-20 [1] CRAN (R 4.1.0)                                      
   utf8                 1.2.2      2021-07-24 [1] CRAN (R 4.1.0)                                      
   vctrs              * 0.3.8      2021-04-29 [1] CRAN (R 4.1.0)                                      
   vip                * 0.3.2      2020-12-17 [1] CRAN (R 4.1.0)                                      
   viridisLite          0.4.0      2021-04-13 [1] CRAN (R 4.1.0)                                      
   withr                2.4.2      2021-04-18 [1] CRAN (R 4.1.0)                                      
   workflows          * 0.2.3      2021-07-16 [1] CRAN (R 4.1.0)                                      
   workflowsets       * 0.1.0      2021-07-22 [1] CRAN (R 4.1.0)                                      
   xgboost            * 1.4.1.1    2021-04-22 [1] CRAN (R 4.1.0)                                      
   xml2                 1.3.2      2020-04-23 [1] CRAN (R 4.1.0)                                      
   xts                  0.12.1     2020-09-09 [1] CRAN (R 4.1.0)                                      
   yaml                 2.2.1      2020-02-01 [1] CRAN (R 4.1.0)                                      
   yardstick          * 0.0.8      2021-03-28 [1] CRAN (R 4.1.0)                                      
   zoo                  1.8-9      2021-03-09 [1] CRAN (R 4.1.0)                                      

[Steviey]

+1

[mdancho84]

For those who stumble up on this question. Then here is a sample code showing how you can do this with a for loop.

library(modeltime)
library(modeltime.ensemble)
library(timetk)
library(tidymodels)

data_tbl <- walmart_sales_weekly %>%
filter(id == "1_3") %>%
select(Date, Weekly_Sales) %>%
set_names("date", "value")

Create splits

splits <- data_tbl %>% time_series_cv(date_var = date, assess = 12, skip = 1, cumulative = TRUE, slice_limit = 5)

initial_fit_data_tbl <- splits$splits[[nrow(splits)]] %>% training()

Create models and ensemble

arima_mtbl <- arima_reg() %>%
set_engine("auto_arima") %>%
fit(value ~ date, data = initial_fit_data_tbl) %>%
modeltime_table()

ets_mtbl <- exp_smoothing() %>%
set_engine("ets") %>%
fit(value ~ date, data = initial_fit_data_tbl) %>%
modeltime_table()

models_mtbl <- combine_modeltime_tables(
arima_mtbl,
ets_mtbl
)

ensemble

models_mtbl <- models_mtbl %>%
ensemble_average() %>%
modeltime_table() %>%
combine_modeltime_tables(models_mtbl)

Cross validate

This does not work

models_mtbl %>%
modeltime_fit_resamples(splits) %>%
modeltime_resample_accuracy()

Use for loop instead

accuracy_ls <- list() # to store the accuracy calculations in

for(i in 1:nrow(splits)) {

print(glue::glue("Resample: {i}"))

# Create train and validation/test data sets
train_data_loop_tbl <- splits$splits[[i]] %>% training()
test_data_loop_tbl  <- splits$splits[[i]] %>% testing()

# Refit model and calculate accuracy
accuracy_tbl <- models_mtbl %>% 
    modeltime_refit(train_data_loop_tbl) %>% 
    modeltime_accuracy(new_data = test_data_loop_tbl) %>% 
    
    # just so I know which resample this accuracy calculations refers too
    # This is not important. Just informative
    mutate(resample = paste0("resample_", i))

accuracy_ls[[i]] <- accuracy_tbl

}

Collect accuracy calculations

accuracy_tbl <- accuracy_ls %>% bind_rows()

Calculate mean accuracy per model and arrange by mean rmse

accuracy_tbl %>%
group_by(.model_desc) %>%
summarise(
mean_rmse = mean(rmse),
mean_mae = mean(mae)
) %>%
arrange(mean_rmse)