diff --git a/.github/workflows/dsBase_test_suite.yaml b/.github/workflows/dsBase_test_suite.yaml new file mode 100644 index 00000000..00b230a4 --- /dev/null +++ b/.github/workflows/dsBase_test_suite.yaml @@ -0,0 +1,210 @@ +################################################################################ +# DataSHIELD GHA test suite - dsBase +# Adapted from `azure-pipelines.yml` by Roberto Villegas-Diaz +# +# Inside the root directory $(Pipeline.Workspace) will be a file tree like: +# /dsBase <- Checked out version of datashield/dsBase +# /dsBase/logs <- Where results of tests and logs are collated +# /testStatus <- Checked out version of datashield/testStatus +# +# As of Jul 2025 this takes ~ 9 mins to run. +################################################################################ +name: dsBase tests' suite + +on: + push: + branches: [master, v6.3.4-dev] + schedule: + - cron: '0 0 * * 0' # Weekly + - cron: '0 1 * * *' # Nightly + +jobs: + dsBase_test_suite: + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: write + + # These should all be constant, except TEST_FILTER. This can be used to test + # subsets of test files in the testthat directory. Options are like: + # '*' <- Run all tests. + # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests. + # '*_smk_*' <- Run all the smoke tests for all functions. + env: + TEST_FILTER: '*' + _r_check_system_clock_: 0 + WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }} + PROJECT_NAME: dsBase + BRANCH_NAME: ${{ github.ref_name }} + REPO_OWNER: ${{ github.repository_owner }} + R_KEEP_PKG_SOURCE: yes + + steps: + - name: Checkout dsBase + uses: actions/checkout@v4 + with: + path: dsBase + + - name: Checkout testStatus + uses: actions/checkout@v4 + with: + repository: ${{ env.REPO_OWNER }}/testStatus + token: ${{ secrets.GH_TOKEN }} + ref: master + path: testStatus + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: | + any::rcmdcheck + cran::devtools + cran::git2r + cran::RCurl + cran::readr + cran::magrittr + cran::xml2 + cran::purrr + cran::dplyr + cran::stringr + cran::tidyr + cran::quarto + cran::knitr + cran::kableExtra + cran::rmarkdown + cran::downlit + needs: check + + - name: Check man files up-to-date + run: | + orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) + R -e "devtools::document()" + new_sum=$(find man -type f | sort -u | xargs cat | md5sum) + if [ "$orig_sum" != "$new_sum" ]; then + echo "Your committed manual files (man/*.Rd) are out of sync with the R files. Run devtools::document() locally then commit." + exit 1 + else + echo "Documentation up-to-date." + fi + working-directory: dsBase + continue-on-error: true + + - name: Run devtools::check + run: | + R -q -e "library('devtools'); devtools::check(args = c('--no-tests', '--no-examples'))" | tee ../check.Rout + grep -q "^0 errors" ../check.Rout && grep -q " 0 warnings" ../check.Rout && grep -q " 0 notes" ../check.Rout + working-directory: dsBase + continue-on-error: true + + - name: Run tests with coverage & JUnit report + run: | + mkdir -p logs + R -q -e "devtools::reload();" + R -q -e ' + write.csv( + covr::coverage_to_list( + covr::package_coverage( + type = c("none"), + code = c('"'"' + output_file <- file("test_console_output.txt"); + sink(output_file); + sink(output_file, type = "message"); + junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml")); + progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); + multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' + ) + ) + ), + "coveragelist.csv" + )' + + mv coveragelist.csv logs/ + mv test_* logs/ + grep -q " FAIL 0 " logs/test_console_output.txt + working-directory: dsBase + + - name: Check for JUnit errors + run: | + issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true) + echo "Number of testsuites with issues: $issue_count" + sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true + cat issues.log || true + exit $issue_count + working-directory: dsBase/logs + + - name: Write versions to file + run: | + echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt + echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt + echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt + working-directory: dsBase/logs + + - name: Parse results from testthat and covr + run: | + Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ + working-directory: dsBase + + - name: Commit results to testStatus + # if: github.repository == 'villegar/dsBase' && github.event_name != 'pull_request' + run: | + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + cd testStatus + git checkout master + git pull + + mkdir -p logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ + # clear the latest directory + rm -rf docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/* + + # Copy logs to new logs directory location + cp -rv ../dsBase/logs/* logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + cp -rv ../dsBase/logs/${{ env.WORKFLOW_ID }}.txt logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + + # Create symbolic links + ln -sf ${{ env.WORKFLOW_ID }}/ logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/.LATEST + # ln -sf docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/latest + + R -e 'input_dir <- file.path("../logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' + mv source/test_report.html docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html + cp -r docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest + + git add . + git commit -m "Auto test for ${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }} @ ${{ env.WORKFLOW_ID }}" + git push + + env: + PROJECT_NAME: ${{ env.PROJECT_NAME }} + BRANCH_NAME: ${{ env.BRANCH_NAME }} + WORKFLOW_ID: ${{ env.WORKFLOW_ID }} + + - name: Dump environment info + run: | + echo -e "\n#############################" + echo -e "ls /: ######################" + ls -al . + echo -e "\n#############################" + echo -e "lscpu: ######################" + lscpu + echo -e "\n#############################" + echo -e "memory: #####################" + free -m + echo -e "\n#############################" + echo -e "env: ########################" + env + echo -e "\n#############################" + echo -e "R sessionInfo(): ############" + R -e 'sessionInfo()' + sudo apt install tree -y + tree . + diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml new file mode 100644 index 00000000..bfc9f4db --- /dev/null +++ b/.github/workflows/pkgdown.yaml @@ -0,0 +1,49 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + release: + types: [published] + workflow_dispatch: + +name: pkgdown.yaml + +permissions: read-all + +jobs: + pkgdown: + runs-on: ubuntu-latest + # Only restrict concurrency for non-PR jobs + concurrency: + group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgdown, local::. + needs: website + + - name: Build site + run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) + shell: Rscript {0} + + - name: Deploy to GitHub pages 🚀 + if: github.event_name != 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4.5.0 + with: + clean: false + branch: gh-pages + folder: docs diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml new file mode 100644 index 00000000..0ab748d6 --- /dev/null +++ b/.github/workflows/test-coverage.yaml @@ -0,0 +1,62 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + +name: test-coverage.yaml + +permissions: read-all + +jobs: + test-coverage: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::covr, any::xml2 + needs: coverage + + - name: Test coverage + run: | + cov <- covr::package_coverage( + quiet = FALSE, + clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") + ) + print(cov) + covr::to_cobertura(cov) + shell: Rscript {0} + + - uses: codecov/codecov-action@v5 + with: + # Fail if error if not on PR, or if on PR and token is given + fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }} + files: ./cobertura.xml + plugins: noop + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Show testthat output + if: always() + run: | + ## -------------------------------------------------------------------- + find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-test-failures + path: ${{ runner.temp }}/package diff --git a/DESCRIPTION b/DESCRIPTION index 1c5c6270..8c7be464 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,23 +1,27 @@ Package: dsBase -Title: 'DataSHIELD' Server Site Base Functions +Title: 'DataSHIELD' Server Side Base Functions Description: Base 'DataSHIELD' functions for the server side. 'DataSHIELD' is a software package which allows you to do non-disclosive federated analysis on sensitive data. 'DataSHIELD' analytic functions have been designed to only share non disclosive summary statistics, with built in automated output checking based on statistical disclosure control. With data sites setting the threshold values for the automated output checks. For more details, see 'citation("dsBase")'. -Version: 6.3.3 +Version: 6.3.4-9000 Authors@R: c(person(given = "Paul", family = "Burton", - role = c("aut")), + role = c("aut"), + comment = c(ORCID = "0000-0001-5799-9634")), person(given = "Rebecca", family = "Wilson", - role = c("aut")), + role = c("aut"), + comment = c(ORCID = "0000-0003-2294-593X")), person(given = "Olly", family = "Butters", - role = c("aut")), + role = c("aut"), + comment = c(ORCID = "0000-0003-0354-8461")), person(given = "Patricia", family = "Ryser-Welch", - role = c("aut")), + role = c("aut"), + comment = c(ORCID = "0000-0002-0070-0264")), person(given = "Alex", family = "Westerberg", role = c("aut")), @@ -37,6 +41,17 @@ Authors@R: c(person(given = "Paul", role = c("aut"), email = "yannick.marcon@obiba.org", comment = c(ORCID = "0000-0003-0138-2023")), + person(given = "Tom", + family = "Bishop", + role = c("aut")), + person(given = "Amadou", + family = "Gaye", + role = c("aut"), + comment = c(ORCID = "0000-0002-1180-2792")), + person(given = "Xavier", + family = "Escribà-Montagut", + role = c("aut"), + comment = c(ORCID = "0000-0003-2888-8948")), person(given = "Stuart", family = "Wheater", role = c("aut", "cre"), diff --git a/R/BooleDS.R b/R/BooleDS.R index 1ad5e14f..0f54dfca 100644 --- a/R/BooleDS.R +++ b/R/BooleDS.R @@ -28,7 +28,7 @@ BooleDS <- function(V1.name=NULL, V2.name=NULL, Boolean.operator.n=NULL, na.assign.text, numeric.output=TRUE){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ######################################################################### # DataSHIELD MODULE: CAPTURE THE nfilter SETTINGS # diff --git a/R/blackBoxDS.R b/R/blackBoxDS.R index 8e7e33f1..a2ac1efc 100644 --- a/R/blackBoxDS.R +++ b/R/blackBoxDS.R @@ -67,8 +67,11 @@ blackBoxDS <- function(input.var.name=NULL, ######################################################## # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) input.var <- eval(parse(text=input.var.name), envir = parent.frame()) diff --git a/R/blackBoxRanksDS.R b/R/blackBoxRanksDS.R index cb2fc21f..27a85778 100644 --- a/R/blackBoxRanksDS.R +++ b/R/blackBoxRanksDS.R @@ -62,8 +62,11 @@ blackBoxRanksDS <- function(input.var.name=NULL, shared.seedval){ #START FUNC ######################################################## # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) input.var <- eval(parse(text=input.var.name), envir = parent.frame()) input.global.ranks<-input.var diff --git a/R/cbindDS.R b/R/cbindDS.R index 5d5464e7..b7864864 100644 --- a/R/cbindDS.R +++ b/R/cbindDS.R @@ -24,7 +24,7 @@ cbindDS <- function(x.names.transmit=NULL, colnames.transmit=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) x.names.input <- x.names.transmit x.names.act1 <- unlist(strsplit(x.names.input, split=",")) diff --git a/R/dataFrameDS.R b/R/dataFrameDS.R index a3e06f4c..7a2b36be 100644 --- a/R/dataFrameDS.R +++ b/R/dataFrameDS.R @@ -40,7 +40,7 @@ dataFrameDS <- function(vectors=NULL, r.names=NULL, ch.rows=FALSE, ch.names=TRUE, clnames=NULL, strAsFactors=TRUE, completeCases=FALSE){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ######################################################################### # DataSHIELD MODULE: CAPTURE THE nfilter SETTINGS diff --git a/R/dataFrameSortDS.R b/R/dataFrameSortDS.R index 0ea3b3c1..a398a70b 100644 --- a/R/dataFrameSortDS.R +++ b/R/dataFrameSortDS.R @@ -36,7 +36,7 @@ dataFrameSortDS <- function(df.name=NULL,sort.key.name=NULL,sort.descending,sort.method){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ######################################################################### # DataSHIELD MODULE: CAPTURE THE nfilter SETTINGS diff --git a/R/dataFrameSubsetDS1.R b/R/dataFrameSubsetDS1.R index 232e405c..4b3c9476 100644 --- a/R/dataFrameSubsetDS1.R +++ b/R/dataFrameSubsetDS1.R @@ -50,7 +50,7 @@ dataFrameSubsetDS1 <- function(df.name=NULL,V1.name=NULL,V2.name=NULL,Boolean.operator.n=NULL,keep.cols=NULL,rm.cols=NULL,keep.NAs=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ######################################################################### # DataSHIELD MODULE: CAPTURE THE nfilter SETTINGS diff --git a/R/dataFrameSubsetDS2.R b/R/dataFrameSubsetDS2.R index 3a59602f..05938a60 100644 --- a/R/dataFrameSubsetDS2.R +++ b/R/dataFrameSubsetDS2.R @@ -59,7 +59,7 @@ dataFrameSubsetDS2<-function(df.name=NULL,V1.name=NULL, V2.name=NULL, Boolean.operator.n=NULL,keep.cols=NULL, rm.cols=NULL, keep.NAs=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ######################################################################### # DataSHIELD MODULE: CAPTURE THE nfilter SETTINGS # diff --git a/R/global.R b/R/global.R index 89970bc0..302a35b1 100644 --- a/R/global.R +++ b/R/global.R @@ -2,5 +2,5 @@ utils::globalVariables(c('offset.to.use', 'weights.to.use', 'out.table.real', 'out.table.dim', 'out.table.dimnames', 'list.obj', 'mg', 'blackbox.output.df', 'blackbox.ranks.df', 'global.bounds.df', 'global.ranks.quantiles.df', 'sR4.df', - 'min.max.df','sR5.df','input.mean.sd.df','input.ranks.sd.df','RS','CG','mixed', 'x','y')) + 'min.max.df','sR5.df','input.mean.sd.df','input.ranks.sd.df','RS','CG','mixed','x','y','.Random.seed')) diff --git a/R/heatmapPlotDS.R b/R/heatmapPlotDS.R index 6dccbd5b..68302392 100644 --- a/R/heatmapPlotDS.R +++ b/R/heatmapPlotDS.R @@ -39,8 +39,11 @@ heatmapPlotDS <- function(x, y, k, noise, method.indicator){ ################################################################### # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) # Cbind the columns of the two variables and remove any rows that include NAs data.table <- cbind.data.frame(x, y) diff --git a/R/histogramDS1.R b/R/histogramDS1.R index a79d2f52..c5f68d41 100644 --- a/R/histogramDS1.R +++ b/R/histogramDS1.R @@ -37,8 +37,11 @@ histogramDS1 <- function(xvect, method.indicator, k, noise){ ################################################################## # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) # print an error message if the input vector is not a numeric if(!(is.numeric(xvect))){ diff --git a/R/histogramDS2.R b/R/histogramDS2.R index 1f7a8acc..cffed190 100644 --- a/R/histogramDS2.R +++ b/R/histogramDS2.R @@ -39,8 +39,11 @@ histogramDS2 <- function (xvect, num.breaks, min, max, method.indicator, k, nois ################################################################## # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) if (method.indicator==1){ diff --git a/R/levelsDS.R b/R/levelsDS.R index 9bb54401..bdb374d5 100644 --- a/R/levelsDS.R +++ b/R/levelsDS.R @@ -10,7 +10,7 @@ levelsDS <- function(x){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ################################################################## #MODULE 1: CAPTURE THE nfilter SETTINGS # diff --git a/R/minMaxRandDS.R b/R/minMaxRandDS.R index 0346eb75..e6ccfc82 100644 --- a/R/minMaxRandDS.R +++ b/R/minMaxRandDS.R @@ -21,8 +21,11 @@ minMaxRandDS <- function(input.var.name){ #START FUNC # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) input.var <- eval(parse(text=input.var.name), envir = parent.frame()) diff --git a/R/rangeDS.R b/R/rangeDS.R index 6cf7689c..9c391b69 100644 --- a/R/rangeDS.R +++ b/R/rangeDS.R @@ -9,8 +9,11 @@ #' rangeDS <- function(xvect) { # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) # print an error message if the input vector is not a numeric if (!(is.numeric(xvect))) { diff --git a/R/reShapeDS.R b/R/reShapeDS.R index 6dbf0654..2ec368a5 100644 --- a/R/reShapeDS.R +++ b/R/reShapeDS.R @@ -43,7 +43,7 @@ reShapeDS <- function(data.name, varying.transmit, v.names.transmit, timevar.name, idvar.name, drop.transmit, direction, sep){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) datatext <- paste0("data.frame(",data.name,")") data <- eval(parse(text=datatext), envir = parent.frame()) diff --git a/R/recodeLevelsDS.R b/R/recodeLevelsDS.R index c773e774..6878faa6 100644 --- a/R/recodeLevelsDS.R +++ b/R/recodeLevelsDS.R @@ -11,7 +11,7 @@ recodeLevelsDS <- function (x=NULL, classes=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) # check if the input vector is valid (i.e. meets DataSHIELD criteria) check <- isValidDS(x) diff --git a/R/recodeValuesDS.R b/R/recodeValuesDS.R index f29dc0fa..d22a7862 100644 --- a/R/recodeValuesDS.R +++ b/R/recodeValuesDS.R @@ -31,7 +31,7 @@ recodeValuesDS <- function(var.name.text=NULL, values2replace.text=NULL, new.values.text=NULL, missing=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) ############################################################# #MODULE 1: CAPTURE THE used nfilter SETTINGS diff --git a/R/scatterPlotDS.R b/R/scatterPlotDS.R index bf229041..62fbc6a6 100644 --- a/R/scatterPlotDS.R +++ b/R/scatterPlotDS.R @@ -44,8 +44,11 @@ scatterPlotDS <- function(x, y, method.indicator, k, noise){ ################################################################### # back-up current .Random.seed and revert on.exit - old_seed <- .Random.seed - on.exit(.Random.seed <- old_seed, add = TRUE) + if (exists(x = ".Random.seed", envir = globalenv())) { + assign(x = "old_seed", value = .Random.seed, envir = parent.frame()); + on.exit({ assign(x = ".Random.seed", value = old_seed, envir = globalenv()); remove("old_seed", envir = parent.frame()) }, add = TRUE) + } else + on.exit(if (exists(x = ".Random.seed", envir = globalenv())) remove(".Random.seed", envir = globalenv()), add = TRUE) # Cbind the columns of the two variables and remove any rows that include NAs data.table <- cbind.data.frame(x, y) diff --git a/R/subsetByClassDS.R b/R/subsetByClassDS.R index f213bb4d..e4063482 100644 --- a/R/subsetByClassDS.R +++ b/R/subsetByClassDS.R @@ -18,7 +18,7 @@ subsetByClassDS <- function(data=NULL, variables=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) # this filter sets the minimum number of observations that are allowed diff --git a/R/subsetDS.R b/R/subsetDS.R index 7ba57ac2..2ab0a14c 100644 --- a/R/subsetDS.R +++ b/R/subsetDS.R @@ -30,7 +30,7 @@ subsetDS <- function(dt=NULL, complt=NULL, rs=NULL, cs=NULL, lg=NULL, th=NULL, varname=NULL){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) # this filter sets the minimum number of observations that are allowed diff --git a/R/vectorDS.R b/R/vectorDS.R index 8efacf97..e1bff36b 100644 --- a/R/vectorDS.R +++ b/R/vectorDS.R @@ -10,7 +10,7 @@ #' vectorDS <- function(...){ # Check Permissive Privacy Control Level. - dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana')) + dsBase::checkPermissivePrivacyControlLevel(c('permissive', 'banana', 'carrot')) # compute the vector's value out <- c(...) diff --git a/R/zzz.R b/R/zzz.R deleted file mode 100644 index 6586ae7c..00000000 --- a/R/zzz.R +++ /dev/null @@ -1,20 +0,0 @@ -ENV <- new.env() - -.onLoad = function(libname, pkgname) { - - #### !!! If making changes, update: .onLoad(), set_opts(), show_opts(), .check_options() - - options( - datashield.privacyLevel = 5, - default.datashield.privacyControlLevel = "banana", - default.nfilter.glm = 0.33, - default.nfilter.kNN = 3, - default.nfilter.string = 80, - default.nfilter.subset = 3, - default.nfilter.stringShort = 20, - default.nfilter.tab = 3, - default.nfilter.noise = 0.25, - default.nfilter.levels.density = 0.33, - default.nfilter.levels.max = 40 - ) -} \ No newline at end of file diff --git a/README.md b/README.md index aac29667..cc983936 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,36 @@ -dsBase -====== +## dsBase: 'DataSHIELD' Server Side Base Functions -DataSHIELD server side base R library. +[![License](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) +[![](https://www.r-pkg.org/badges/version/dsBase?color=black)](https://cran.r-project.org/package=dsBase) +[![R build +status](https://github.com/datashield/dsBase/workflows/R-CMD-check/badge.svg)](https://github.com/datashield/dsBase/actions) +[![Codecov test coverage](https://codecov.io/gh/datashield/dsBase/graph/badge.svg)](https://app.codecov.io/gh/datashield/dsBase) -[![License](https://img.shields.io/badge/license-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html) +## Installation -About -===== +You can install the released version of dsBase from +[CRAN](https://cran.r-project.org/package=dsBase) with: + +``` r +install.packages("dsBase") +``` + +And the development version from +[GitHub](https://github.com/datashield/dsBase/) with: + + +``` r +install.packages("remotes") +remotes::install_github("datashield/dsBase", "") + +# Install v6.4.0 with the following +remotes::install_github("datashield/dsBase", "v6.4.0-dev") +``` + +For a full list of development branches, checkout https://github.com/datashield/dsBase/branches + + +## About DataSHIELD is a software package which allows you to do non-disclosive federated analysis on sensitive data. Our website (https://www.datashield.org) has in depth descriptions of what it is, how it works and how to install it. A key point to highlight is that DataSHIELD has a client-server infrastructure, so the dsBase package (https://github.com/datashield/dsBase) needs to be used in conjuction with the dsBaseClient package (https://github.com/datashield/dsBaseClient) - trying to use one without the other makes no sense. @@ -18,3 +42,30 @@ Detailed instructions on how to install DataSHIELD are at https://wiki.datashiel | obiba CRAN | Where you probably should install DataSHIELD from. | | releases | Stable releases. | | master branch | Mostly in sync with the latest release, changes rarely. | + +## References + +[1] Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, + Avraam D, Marcon Y, Bishop T, Gaye A, Escribà Montagut X, Wheater S (2025). _dsBase: 'DataSHIELD' Server Side Base Functions_. R + package version 6.3.3. . + +[2] Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio + M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, + Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, + Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, + Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, + Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). + “DataSHIELD: taking the analysis to the data, not the data to the analysis.” _International + Journal of Epidemiology_, *43*(6), 1929-1944. . + +[3] Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). + “DataSHIELD – New Directions and Dimensions.” _Data Science Journal_, *16*(21), 1-21. + . + +[4] Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, + Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, + Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, + Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, + Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). “DataSHIELD: + mitigating disclosure risk in a multi-site federated analysis platform.” _Bioinformatics + Advances_, *5*(1), 1-21. . diff --git a/docs/authors.html b/docs/authors.html index 345f4351..70570f2a 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -88,16 +88,62 @@

Citation

-

Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Wheater S (2025). +

Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Wheater S (????). dsBase: 'DataSHIELD' Server Site Base Functions. R package version 6.3.3. 

@Manual{,
   title = {dsBase: 'DataSHIELD' Server Site Base Functions},
   author = {Paul Burton and Rebecca Wilson and Olly Butters and Patricia Ryser-Welch and Alex Westerberg and Leire Abarrategui and Roberto Villegas-Diaz and Demetris Avraam and Yannick Marcon and Stuart Wheater},
-  year = {2025},
   note = {R package version 6.3.3},
 }
+

Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). +“DataSHIELD: taking the analysis to the data, not the data to the analysis.” +International Journal of Epidemiology, 43(6), 1929–1944. +doi:10.1093/ije/dyu188. +

+ 
@Article{,
+  title = {{DataSHIELD: taking the analysis to the data, not the data to the analysis}},
+  author = {Amadou Gaye and Yannick Marcon and Julia Isaeva and Philippe {LaFlamme} and Andrew Turner and Elinor M Jones and Joel Minion and Andrew W Boyd and Christopher J Newby and Marja-Liisa Nuotio and Rebecca Wilson and Oliver Butters and Barnaby Murtagh and Ipek Demir and Dany Doiron and Lisette Giepmans and Susan E Wallace and Isabelle Budin-Lj{\o}sne and Carsten {{Oliver Schmidt}} and Paolo Boffetta and Mathieu Boniol and Maria Bota and Kim W Carter and Nick {deKlerk} and Chris Dibben and Richard W Francis and Tero Hiekkalinna and Kristian Hveem and Kirsti Kval{\o}y and Sean Millar and Ivan J Perry and Annette Peters and Catherine M Phillips and Frank Popham and Gillian Raab and Eva Reischl and Nuala Sheehan and Melanie Waldenberger and Markus Perola and Edwin {{van den Heuvel}} and John Macleod and Bartha M Knoppers and Ronald P Stolk and Isabel Fortier and Jennifer R Harris and Bruce H R Woffenbuttel and Madeleine J Murtagh and Vincent Ferretti and Paul R Burton},
+  journal = {International Journal of Epidemiology},
+  year = {2014},
+  volume = {43},
+  number = {6},
+  pages = {1929--1944},
+  doi = {10.1093/ije/dyu188},
+}
+

Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). +“DataSHIELD – New Directions and Dimensions.” +Data Science Journal, 16(21), 1–21. +doi:10.5334/dsj-2017-021. +

+ 
@Article{,
+  title = {{DataSHIELD – New Directions and Dimensions}},
+  author = {Rebecca C. Wilson and Oliver {W. Butters} and Demetris Avraam and James Baker and Jonathan A. Tedds and Andrew Turner and Madeleine Murtagh and Paul {R. Burton}},
+  journal = {Data Science Journal},
+  year = {2017},
+  volume = {16},
+  number = {21},
+  pages = {1--21},
+  doi = {10.5334/dsj-2017-021},
+}
+

Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). +“DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform.” +Bioinformatics Advances, 5(1), 1–21. +doi:10.1093/bioadv/vbaf046. +

+ 
@Article{,
+  title = {{DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform}},
+  author = {Demetris Avraam and Rebecca C Wilson and Noemi {{Aguirre Chan}} and Soumya Banerjee and Tom R P Bishop and Olly Butters and Tim Cadman and Luise Cederkvist and Liesbeth Duijts and Xavier {{Escrib{\a`a} Montagut}} and Hugh Garner and Gon{\c c}alo {Gon{\c c}alves} and Juan R Gonz{\a'a}lez and Sido Haakma and Mette Hartlev and Jan Hasenauer and Manuel Huth and Eleanor Hyde and Vincent W V Jaddoe and Yannick Marcon and Michaela Th Mayrhofer and Fruzsina Molnar-Gabor and Andrei Scott Morgan and Madeleine Murtagh and Marc Nestor and Anne-Marie {{Nybo Andersen}} and Simon Parker and Angela {{Pinot de Moira}} and Florian Schwarz and Katrine Strandberg-Larsen and {Morris AvSwertz} and Marieke Welten and Stuart Wheater and Paul Burton},
+  journal = {Bioinformatics Advances},
+  year = {2024},
+  volume = {5},
+  number = {1},
+  pages = {1--21},
+  doi = {10.1093/bioadv/vbaf046},
+  editor = {Thomas Lengauer},
+  publisher = {Oxford University Press (OUP)},
+}
diff --git a/docs/index.html b/docs/index.html index 54bbe6e5..ca9ac5a7 100644 --- a/docs/index.html +++ b/docs/index.html @@ -12,7 +12,7 @@ - +

+
+install.packages("remotes")
+remotes::install_github("datashield/dsBase", "<BRANCH>")
+
+# Install v6.4.0 with the following
+remotes::install_github("datashield/dsBase", "v6.4.0-dev")
+

For a full list of development branches, checkout https://github.com/datashield/dsBase/branches

+ +
+

About +

DataSHIELD is a software package which allows you to do non-disclosive federated analysis on sensitive data. Our website (https://www.datashield.org) has in depth descriptions of what it is, how it works and how to install it. A key point to highlight is that DataSHIELD has a client-server infrastructure, so the dsBase package (https://github.com/datashield/dsBase) needs to be used in conjuction with the dsBaseClient package (https://github.com/datashield/dsBaseClient) - trying to use one without the other makes no sense.

Detailed instructions on how to install DataSHIELD are at https://wiki.datashield.org/. The code here is organised as:

@@ -72,31 +86,46 @@

About

- + - + - + - +
Location What is it?
obiba CRAN Where you probably should install DataSHIELD from.
releases Stable releases.
master branch Mostly in sync with the latest release, changes rarely.
+
+
+

References +

+

[1] Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R, Avraam D, Marcon Y, Bishop T, Gaye A, Escribà Montagut X, Wheater S (2025). dsBase: ‘DataSHIELD’ Server Side Base Functions. R package version 6.3.3. https://doi.org/10.32614/CRAN.package.dsBase.

+

[2] Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I, Oliver Schmidt C, Boffetta P, Boniol M, Bota M, Carter K, deKlerk N, Dibben C, Francis R, Hiekkalinna T, Hveem K, Kvaløy K, Millar S, Perry I, Peters A, Phillips C, Popham F, Raab G, Reischl E, Sheehan N, Waldenberger M, Perola M, van den Heuvel E, Macleod J, Knoppers B, Stolk R, Fortier I, Harris J, Woffenbuttel B, Murtagh M, Ferretti V, Burton P (2014). “DataSHIELD: taking the analysis to the data, not the data to the analysis.” International Journal of Epidemiology, 43(6), 1929-1944. https://doi.org/10.1093/ije/dyu188.

+

[3] Wilson R, W. Butters O, Avraam D, Baker J, Tedds J, Turner A, Murtagh M, R. Burton P (2017). “DataSHIELD – New Directions and Dimensions.” Data Science Journal, 16(21), 1-21. https://doi.org/10.5334/dsj-2017-021.

+

[4] Avraam D, Wilson R, Aguirre Chan N, Banerjee S, Bishop T, Butters O, Cadman T, Cederkvist L, Duijts L, Escribà Montagut X, Garner H, Gonçalves G, González J, Haakma S, Hartlev M, Hasenauer J, Huth M, Hyde E, Jaddoe V, Marcon Y, Mayrhofer M, Molnar-Gabor F, Morgan A, Murtagh M, Nestor M, Nybo Andersen A, Parker S, Pinot de Moira A, Schwarz F, Strandberg-Larsen K, Morris AvSwertz, Welten M, Wheater S, Burton P (2024). “DataSHIELD: mitigating disclosure risk in a multi-site federated analysis platform.” Bioinformatics Advances, 5(1), 1-21. https://doi.org/10.1093/bioadv/vbaf046.