[help] abort for recurvise errors?

[stemangiola]

Help

• I understand and agree to https://books.ropensci.org/targets/help.html.

Description

Hello, I get this error, for which the pipeline crashes, and if I restart it, it executes other ~500 jobs and crashes again.

some of the error (INTEGER) comes as false positive along with other errors from R4.4.0, I think the only useful info here is "no more error handlers available ". What does this point to I don't know.

Is because some errored 4 times and I have tasks_max = 5 in my resources?

queued | skipped | dispatched | completed | errored | warned | canceled | time       
...
...
...
252    | 11964   | 5          | 902       | 4       | 0      | 0        | 10:16 03.06Submitted batch job 17373333
251    | 11964   | 5          | 903       | 4       | 0      | 0        | 10:16 05.71Submitted batch job 17373334
249    | 11964   | 5          | 905       | 4       | 0      | 0        | 10:17 35.45Submitted batch job 17373337
241    | 11964   | 5          | 913       | 4       | 0      | 0        | 10:24 02.67Submitted batch job 17373348
235    | 11964   | 5          | 919       | 4       | 0      | 0        | 10:32 11.21Submitted batch job 17373411
229    | 11964   | 5          | 925       | 4       | 0      | 0        | 10:36 17.65Submitted batch job 17373427
226    | 11964   | 5          | 928       | 4       | 0      | 0        | 10:37 02.44Submitted batch job 17373434
221    | 11964   | 5          | 933       | 4       | 0      | 0        | 10:39 38.06Submitted batch job 17373439
215    | 11964   | 5          | 939       | 4       | 0      | 0        | 10:45 27.59Submitted batch job 17373468
213    | 11964   | 5          | 941       | 4       | 0      | 0        | 10:48 04.44Submitted batch job 17373485
211    | 11964   | 5          | 943       | 4       | 0      | 0        | 10:48 53.47Error: no more error handlers available (recursive errors?); invoking 'abort' restart
Error in stopifnot(length(class2) == 1L) : 
  INTEGER() can only be applied to a 'integer', not a 'unknown type #29'
In addition: Warning message:
In stopifnot(length(class2) == 1L) :
  type 29 is unimplemented in 'type2char'

The pipeline is below. It is quite hard to do a repex in this case because it does not have a deterministic crash

Pipeline

library(targets)
library(tidyverse)
library(targets)
library(tarchetypes)
library(glue)
library(qs)
library(crew)
library(crew.cluster)
tar_option_set(packages = c("CuratedAtlasQueryR", "stringr", 
    "tibble", "tidySingleCellExperiment", "dplyr", "Matrix", 
    "Seurat", "glue", "qs", "purrr", "tidybulk", "tidySummarizedExperiment", 
    "edgeR", "crew", "magrittr", "digest", "glmmSeq", "readr", 
    "forcats"), garbage_collection = TRUE, memory = "transient", 
    storage = "worker", retrieval = "worker", error = "continue", 
    format = "qs", controller = crew_controller_group(crew_controller_slurm(name = "slurm_1_5", 
        slurm_memory_gigabytes_per_cpu = 5, slurm_cpus_per_task = 1, 
        workers = 200, tasks_max = 5, verbose = T), crew_controller_slurm(name = "slurm_1_10", 
        slurm_memory_gigabytes_per_cpu = 10, slurm_cpus_per_task = 1, 
        workers = 100, tasks_max = 5, verbose = T), crew_controller_slurm(name = "slurm_1_20", 
        slurm_memory_gigabytes_per_cpu = 20, slurm_cpus_per_task = 1, 
        workers = 60, tasks_max = 5, verbose = T), crew_controller_slurm(name = "slurm_1_40", 
        slurm_memory_gigabytes_per_cpu = 40, slurm_cpus_per_task = 1, 
        workers = 30, tasks_max = 5, verbose = T), crew_controller_slurm(name = "slurm_1_80", 
        slurm_memory_gigabytes_per_cpu = 80, slurm_cpus_per_task = 1, 
        workers = 5, tasks_max = 5, verbose = T), crew_controller_slurm(name = "slurm_2_400", 
        slurm_memory_gigabytes_per_cpu = 400, slurm_cpus_per_task = 2, 
        workers = 1, tasks_max = 5, verbose = T)), resources = tar_resources(crew = tar_resources_crew("slurm_2_20")))
split_metadata = function() {
    CAQ_directory = "/vast/projects/cellxgene_curated"
    my_metadata = mutate(nest(mutate(as_tibble(select(filter(get_metadata(cache_directory = CAQ_directory), 
        collection_id != "c114c20f-1ef4-49a5-9c2e-d965787fb90c"), 
        file_id, sample_, cell_, file_id_db, cell_type_harmonised)), 
        chunk = sample_), data = -c(chunk, file_id)), number_of_cells = map_int(data, 
        nrow))
    my_sci_seq = mutate(nest(mutate(select(unnest(mutate(mutate(nest(as_tibble(select(filter(get_metadata(cache_directory = CAQ_directory), 
        collection_id == "c114c20f-1ef4-49a5-9c2e-d965787fb90c"), 
        file_id, sample_, cell_, file_id_db, cell_type_harmonised)), 
        data = -c(file_id, sample_, cell_type_harmonised)), n = map_int(data, 
        nrow)), data = map2(data, n, ~slice_sample(.x, n = min(.y, 
        10000), replace = FALSE))), data), -n), chunk = sample_), 
        data = -c(chunk, file_id)), number_of_cells = map_int(data, 
        nrow))
    bind_rows(my_metadata, my_sci_seq)
}
get_sce = function(tissue_cell_type_metadata, sce_rownames) {
    cache.path = "/vast/projects/mangiola_immune_map/caq_cloud3"
    dir.create(cache.path, recursive = TRUE, showWarnings = FALSE)
    mutate(tissue_cell_type_metadata, data = pmap(list(data), 
        ~get_single_cell_experiment(..1, cache_directory = cache.path)))
}
get_pseudobulk = function(sce_df) {
    mutate(sce_df, data = map(data, ~{
        .x = tidySingleCellExperiment::aggregate_cells(.x, .sample = c(sample_, 
            cell_type_harmonised))
        my_assay = as(assay(.x, "counts"), "sparseMatrix")
        rownames(my_assay) = NULL
        SummarizedExperiment(assays = list(counts = my_assay), 
            colData = colData(.x))
    }))
}
aggregate = function(se_df, se_rownames) {
    print("Start aggregate")
    gc()
    mutate(nest(mutate(se_df, data = map2(data, file_id, ~{
        se = select(mutate(.x, file_id = .y), -any_of(c("file_id_db", 
            ".cell", "original_cell_id")))
        se
    }, .progress = TRUE)), data = -file_id), data = map(data, 
        ~{
            se = do.call(cbind, args = pull(.x, data))
            se@assays@data$counts = as(se@assays@data$counts, 
                "sparseMatrix")
            rownames(se) = se_rownames
            se
        }))
}
list(
  tar_target(
    sce_rownames, 
    rownames(get_single_cell_experiment(head(get_metadata(), 1))),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tar_target(
    metadata_split, 
    split_metadata(),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_40"))
  ),
  
  tarchetypes::tar_group_by(
    metadata_split_SMALL, 
    metadata_split[metadata_split$number_of_cells < 2500, ], 
    chunk, 
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tarchetypes::tar_group_by(
    metadata_split_BIG, 
    metadata_split[metadata_split$number_of_cells >= 2500, ], 
    chunk, 
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tar_target(
    metadata_split_pseudobulk_SMALL_2, 
    get_pseudobulk(get_sce(metadata_split_SMALL, sce_rownames)),
    pattern = map(metadata_split_SMALL),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tar_target(
    metadata_split_pseudobulk_BIG, 
    get_pseudobulk(get_sce(metadata_split_BIG, sce_rownames)),
    pattern = map(metadata_split_BIG),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_80"))
  ),
  
  tarchetypes::tar_group_by(
    metadata_grouped_pseudobulk_2, 
    bind_rows(metadata_split_pseudobulk_SMALL_2, metadata_split_pseudobulk_BIG),
    file_id,
    resources = tar_resources(crew = tar_resources_crew("slurm_2_400"))
  ),
  
  tar_target(
    pseudobulk_grouped_by_file_id_2, 
    aggregate(metadata_grouped_pseudobulk_2, sce_rownames),
    pattern = map(metadata_grouped_pseudobulk_2),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tar_target(
    the_best_target_distribution, 
    preprocessCore::normalize.quantiles.determine.target(
      as.matrix(assay(pull(filter(pseudobulk_grouped_by_file_id_2, file_id == "c0dca32e-fa64-4448-bc12-2b8f16702c29"), data)[[1]], "counts"))
    ),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  ),
  
  tar_target(
    pseudobulk_file_id_quantile_normalised_3, 
    mutate(pseudobulk_grouped_by_file_id_2, 
      data = map(data, ~select(
        quantile_normalise_abundance(.x, 
          method = "preprocesscore_normalize_quantiles_use_target", 
          target_distribution = the_best_target_distribution
        ), 
        -counts
      ))
    ),
    pattern = map(pseudobulk_grouped_by_file_id_2),
    resources = tar_resources(crew = tar_resources_crew("slurm_1_20"))
  )
)

Sesisoninfo

sessionInfo()
R version 4.4.0 (2024-04-24)
Platform: x86_64-pc-linux-gnu
Running under: CentOS Linux 7 (Core)

Matrix products: default
BLAS:   /stornext/System/data/apps/R/R-4.4.0/lib64/R/lib/libRblas.so 
LAPACK: /stornext/System/data/apps/R/R-4.4.0/lib64/R/lib/libRlapack.so;  LAPACK version 3.12.0

locale:
 [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C               LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8     LC_MONETARY=en_US.UTF-8   
 [6] LC_MESSAGES=en_US.UTF-8    LC_PAPER=en_US.UTF-8       LC_NAME=C                  LC_ADDRESS=C               LC_TELEPHONE=C            
[11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C       

time zone: UTC
tzcode source: system (glibc)

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] CuratedAtlasQueryR_1.3.7 glue_1.7.0               targets_1.7.1            lubridate_1.9.3          forcats_1.0.0           
 [6] stringr_1.5.1            dplyr_1.1.4              purrr_1.0.2              readr_2.1.5              tidyr_1.3.1             
[11] tibble_3.2.1             ggplot2_3.5.1            tidyverse_2.0.0         

loaded via a namespace (and not attached):
  [1] RcppAnnoy_0.0.22            splines_4.4.0               later_1.3.2                 polyclip_1.10-6            
  [5] preprocessCore_1.66.0       fastDummies_1.7.3           lifecycle_1.0.4             StanHeaders_2.32.7         
  [9] globals_0.16.3              processx_3.8.4              lattice_0.22-6              MASS_7.3-60.2              
 [13] backports_1.5.0             magrittr_2.0.3              plotly_4.10.4               yaml_2.3.8                 
 [17] httpuv_1.6.15               Seurat_5.1.0                sctransform_0.4.1           spam_2.10-0                
 [21] spatstat.sparse_3.0-3       sp_2.1-4                    pkgbuild_1.4.4              reticulate_1.36.1          
 [25] cowplot_1.1.3               pbapply_1.7-2               DBI_1.2.2                   RColorBrewer_1.1-3         
 [29] abind_1.4-5                 zlibbioc_1.50.0             Rtsne_0.17                  GenomicRanges_1.56.0       
 [33] BiocGenerics_0.50.0         GenomeInfoDbData_1.2.12     IRanges_2.38.0              S4Vectors_0.42.0           
 [37] ggrepel_0.9.5               inline_0.3.19               irlba_2.3.5.1               listenv_0.9.1              
 [41] spatstat.utils_3.0-4        goftest_1.2-3               RSpectra_0.16-1             spatstat.random_3.2-3      
 [45] fitdistrplus_1.1-11         parallelly_1.37.1           leiden_0.4.3.1              codetools_0.2-20           
 [49] DelayedArray_0.30.1         RApiSerialize_0.1.3         tidyselect_1.2.1            UCSC.utils_1.0.0           
 [53] matrixStats_1.3.0           stats4_4.4.0                spatstat.explore_3.2-7      duckdb_0.10.2              
 [57] jsonlite_1.8.8              progressr_0.14.0            ggridges_0.5.6              survival_3.6-4             
 [61] tools_4.4.0                 ica_1.0-3                   Rcpp_1.0.12                 gridExtra_2.3              
 [65] SparseArray_1.4.3           xfun_0.44                   MatrixGenerics_1.16.0       GenomeInfoDb_1.40.0        
 [69] HDF5Array_1.32.0            loo_2.7.0                   withr_3.0.0                 fastmap_1.2.0              
 [73] rhdf5filters_1.16.0         fansi_1.0.6                 ttservice_0.4.1             callr_3.7.6                
 [77] digest_0.6.35               secretbase_0.5.0            timechange_0.3.0            R6_2.5.1                   
 [81] mime_0.12                   colorspace_2.1-0            scattermore_1.2             tensor_1.5                 
 [85] spatstat.data_3.0-4         utf8_1.2.4                  generics_0.1.3              data.table_1.15.4          
 [89] httr_1.4.7                  htmlwidgets_1.6.4           S4Arrays_1.4.0              uwot_0.2.2                 
 [93] pkgconfig_2.0.3             gtable_0.3.5                blob_1.2.4                  lmtest_0.9-40              
 [97] SingleCellExperiment_1.26.0 XVector_0.44.0              tarchetypes_0.9.0           htmltools_0.5.8.1          
[101] dotCall64_1.1-1             base64url_1.4               SeuratObject_5.0.2          scales_1.3.0               
[105] Biobase_2.64.0              png_0.1-8                   knitr_1.46                  rstudioapi_0.16.0          
[109] tzdb_0.4.0                  reshape2_1.4.4              checkmate_2.3.1             nlme_3.1-164               
[113] curl_5.2.1                  zoo_1.8-12                  rhdf5_2.48.0                KernSmooth_2.23-24         
[117] parallel_4.4.0              miniUI_0.1.1.1              pillar_1.9.0                grid_4.4.0                 
[121] vctrs_0.6.5                 RANN_2.6.1                  promises_1.3.0              stringfish_0.16.0          
[125] dbplyr_2.5.0                xtable_1.8-4                cluster_2.1.6               cli_3.6.2                  
[129] compiler_4.4.0              rlang_1.1.4                 crayon_1.5.2                future.apply_1.11.2        
[133] tidybulk_1.17.3             ps_1.7.6                    fs_1.6.4                    plyr_1.8.9                 
[137] stringi_1.8.4               rstan_2.32.6                deldir_2.0-4                viridisLite_0.4.2          
[141] QuickJSR_1.1.3              assertthat_0.2.1            munsell_0.5.1               lazyeval_0.2.2             
[145] spatstat.geom_3.2-9         V8_4.4.2                    Matrix_1.7-0                RcppHNSW_0.6.0             
[149] hms_1.1.3                   patchwork_1.2.0             future_1.33.2               Rhdf5lib_1.26.0            
[153] shiny_1.8.1.1               SummarizedExperiment_1.34.0 qs_0.26.3                   ROCR_1.0-11                
[157] igraph_2.0.3                RcppParallel_5.1.7   

[wlandau]

Sounds like a low-level issue, possibly in one of the packages you are using, or an interaction with the RStudio IDE. Possibly relevant: tidyverse/vroom#389, https://stackoverflow.com/questions/78494337/r-debugonce-error-error-no-more-error-handlers-available-recursive-errors

[multimeric]

Yes, this is just an issue we have on our cluster that's related to the old RStudio version we're running. I don't think targets is at fault here. As far as I know, "no more error handlers available" doesn't relate to targets, but is just a base R thing about error handling.