Merge pull request #622 from datashield/v6.3.5-dev

Updates, including context(...) fix

Author: StuartWheater

.Rbuildignore

@@ -17,8 +17,8 @@
 ^R/secure.global.ranking.md$
 ^_pkgdown\.yml$
 ^docs$
-^dsBase_6.3.4.tar.gz$
-^dsBase_6.3.4-permissive.tar.gz$
+^dsBase_6.3.5.tar.gz$
+^dsBase_6.3.5-permissive.tar.gz$
 ^dsDanger_6.3.4.tar.gz$
 ^\.circleci$
 ^\.circleci/config\.yml$

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: dsBaseClient
 Title: 'DataSHIELD' Client Side Base Functions
-Version: 6.3.4
+Version: 6.3.5-9000
 Description: Base 'DataSHIELD' functions for the client side. 'DataSHIELD' is a software package which allows
     you to do non-disclosive federated analysis on sensitive data. 'DataSHIELD' analytic functions have
     been designed to only share non disclosive summary statistics, with built in automated output

README.md

@@ -23,8 +23,8 @@ And the development version from
 install.packages("remotes")
 remotes::install_github("datashield/dsBaseClient", "<BRANCH>")
 
-# Install v6.3.4 with the following
-remotes::install_github("datashield/dsBaseClient", "6.3.4")
+# Install v6.3.5 with the following
+remotes::install_github("datashield/dsBaseClient", "6.3.5")
 ```
 
 For a full list of development branches, checkout https://github.com/datashield/dsBaseClient/branches
@@ -50,7 +50,7 @@ The code here is organised as:
 
 [1] Burton P, Wilson R, Butters O, Ryser-Welch P, Westerberg A, Abarrategui L, Villegas-Diaz R,
   Avraam D, Marcon Y, Bishop T, Gaye A, Escribà Montagut X, Wheater S (2025). 
-  _dsBaseClient: 'DataSHIELD' Client Side Base Functions_. R package version 6.3.4.
+  _dsBaseClient: 'DataSHIELD' Client Side Base Functions_. R package version 6.3.5.
 
 [2] Gaye A, Marcon Y, Isaeva J, LaFlamme P, Turner A, Jones E, Minion J, Boyd A, Newby C, Nuotio
   M, Wilson R, Butters O, Murtagh B, Demir I, Doiron D, Giepmans L, Wallace S, Budin-Ljøsne I,

armadillo_azure-pipelines.yml

@@ -58,10 +58,10 @@ schedules:
       - master
     always: true
   - cron: "0 2 * * *"
-    displayName: Nightly build - v6.3.4-dev
+    displayName: Nightly build - v6.3.5-dev
     branches:
       include:
-      - v6.3.4-dev
+      - v6.3.5-dev
     always: true
 
 #########################################################################################
@@ -235,7 +235,7 @@ jobs:
 
       curl -u admin:admin -X GET http://localhost:8080/packages
 
-      curl -u admin:admin --max-time 300 -v -H 'Content-Type: multipart/form-data' -F "file=@dsBase_6.3.4-permissive.tar.gz" -X POST http://localhost:8080/install-package
+      curl -u admin:admin --max-time 300 -v -H 'Content-Type: multipart/form-data' -F "file=@dsBase_6.3.5-permissive.tar.gz" -X POST http://localhost:8080/install-package
       sleep 60
 
       docker container restart dsbaseclient_armadillo_1

azure-pipelines.yml

@@ -3,11 +3,9 @@
 # Starts with a vanilla Opal docker composition, installs dsBase
 # and dsBaseClient (as well as dependencies - including a fully functional
 # Opal server).
-# Does checks and tests then saves results to testStatus repo.
 #
 # Inside the root directory $(Pipeline.Workspace) will be a file tree like:
 # /dsBaseClient               <- Checked out version of datashield/dsBaseClient
-# /testStatus                 <- Checked out version of datashield/testStatus
 # /logs                       <- Where results of tests and lots are collated
 #
 # As of May 2020 this takes ~ 70 mins to run.
@@ -36,18 +34,6 @@ variables:
   _r_check_system_clock_: 0
 
 
-#########################################################################################
-# Need to define all the GH repos and their access tokens, see:
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml
-resources:
-  repositories:
-  - repository: testStatusRepo
-    type: github
-    endpoint: datashield-testing
-    name: datashield/testStatus
-    ref: master
-
-
 #########################################################################################
 # When and under what condition to run the pipeline.
 schedules:
@@ -58,10 +44,10 @@ schedules:
       - master
     always: true
   - cron: "0 2 * * *"
-    displayName: Nightly build - v6.3.4-dev
+    displayName: Nightly build - v6.3.5-dev
     branches:
       include:
-      - v6.3.4-dev
+      - v6.3.5-dev
     always: true
 
 #########################################################################################
@@ -82,11 +68,6 @@ jobs:
   - checkout: self
     path: 'dsBaseClient'
 
-  - checkout: testStatusRepo
-    path: 'testStatus'
-    persistCredentials: true
-    condition: and(eq(variables['Build.Repository.Name'], 'datashield/dsBaseClient'), ne(variables['Build.Reason'], 'PullRequest'))
-
 
     #####################################################################################
     # The MySQL install that comes with the VM doesn't seem compatable with our set up
@@ -235,7 +216,7 @@ jobs:
   - bash: |
       R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'https://localhost:8443', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(o)"
 
-      R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='https://localhost:8443/', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = 'v6.3.4-dev'); opal.logout(opal)"
+      R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='https://localhost:8443/', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = 'v6.3.5-dev'); opal.logout(opal)"
 
       sleep 60
 
@@ -524,75 +505,6 @@ jobs:
     displayName: 'Down Opal Docker Composition'
     condition: succeeded()
 
-    #####################################################################################
-    #                                Windup phase
-    #####################################################################################
-
-    #####################################################################################
-    # Output some important version numbers to file. This gets added to the testStatus 
-    # commit so it can be parsed and used on the status table.
-  - bash: |
-
-      echo 'branch:'$(branchName) >> $(datetime).txt
-      echo 'os:'$(lsb_release -ds) >> $(datetime).txt
-      echo 'R:'$(R --version | head -n 1) >> $(datetime).txt
-      echo 'opal:'$(opal system --opal localhost:8443 --user administrator --password "datashield_test&" --version) >> $(datetime).txt
-
-    workingDirectory: $(Pipeline.Workspace)/logs
-    displayName: 'Write versions to file'
-    condition: succeededOrFailed()
-
-
-    #####################################################################################
-    # Checkout the testStatus repo, add the results from here, push back to GH.
-    # TODO: Automatically pull in better email/name info from somewhere.
-    # TODO: More debug info in commit message
-  - bash: |
-
-      # Git needs some config set to be able to push to a repo. 
-      git config --global user.email "you@example.com"
-      git config --global user.name "Azure pipeline"
-
-      # This repo is checked out in detatched head state, so reconnect it here.
-      git checkout master
-
-      # It is possible that other commits have been made to the testStatus repo since it 
-      # was checked out. i.e. other pipeline runs might have finished.
-      git pull
-
-      # Make the directories if they dont already exist
-      mkdir --parents logs/$(projectName)/$(branchName)
-      mkdir --parents docs/$(projectName)/$(branchName)/latest
-
-      cp $(Pipeline.Workspace)/logs/coveragelist.csv logs/$(projectName)/$(branchName)/
-      cp $(Pipeline.Workspace)/logs/coveragelist.csv logs/$(projectName)/$(branchName)/$(datetime).csv
-
-      cp $(Pipeline.Workspace)/logs/test_results.xml logs/$(projectName)/$(branchName)/
-      cp $(Pipeline.Workspace)/logs/test_results.xml logs/$(projectName)/$(branchName)/$(datetime).xml
-
-      cp $(Pipeline.Workspace)/logs/$(datetime).txt logs/$(projectName)/$(branchName)/
-
-      # Run the script to parse the results and build the html pages.
-      # status.py JUnit_file.xml coverage_file.csv output_file.html local_repo_path remote_repo_name branch
-      source/status.py logs/$(projectName)/$(branchName)/$(datetime).xml logs/$(projectName)/$(branchName)/$(datetime).csv logs/$(projectName)/$(branchName)/$(datetime).txt status.html $(Pipeline.Workspace)/$(projectName) $(projectName) $(branchName)
-
-      cp status.html docs/$(projectName)/$(branchName)/latest/index.html
-      git add logs/$(projectName)/$(branchName)/coveragelist.csv
-      git add logs/$(projectName)/$(branchName)/test_results.xml
-      git add logs/$(projectName)/$(branchName)/$(datetime).xml
-      git add logs/$(projectName)/$(branchName)/$(datetime).csv
-      git add logs/$(projectName)/$(branchName)/$(datetime).txt
-      git add docs/$(projectName)/$(branchName)/latest/index.html
-
-      git commit -m "Azure auto test for $(projectName)/$(branchName) @ $(datetime)" -m "Debug info:\nProjectName:$(projectName)\nBranchName:$(branchName)\nDataTime:$(datetime)"
-      git push
-      exit 0
-
-    workingDirectory: $(Pipeline.Workspace)/testStatus
-    displayName: 'Parse test results'
-    condition: and(eq(variables['Build.Repository.Name'], 'datashield/dsBaseClient'), ne(variables['Build.Reason'], 'PullRequest'))
-
-
     #####################################################################################
     # Output the environment information to the console. This is useful for debugging.
     # Always do this, even if some of the above has failed or the job has been cacelled.

docker-compose_armadillo.yml

@@ -3,7 +3,7 @@ services:
         hostname: armadillo
         ports:
             - 8080:8080
-        image: datashield/armadillo_citest:latest
+        image: datashield/armadillo_citest:5.9.4
         environment:
             LOGGING_CONFIG: 'classpath:logback-file.xml'
             AUDIT_LOG_PATH: '/app/logs/audit.log'

docker-compose_opal.yml

@@ -20,6 +20,6 @@ services:
             - MONGO_INITDB_ROOT_USERNAME=root
             - MONGO_INITDB_ROOT_PASSWORD=foobar
     rock:
-        image: datashield/rock-omicron-karma-permissive:devel
+        image: datashield/rock-lemon-donkey-permissive:draft
         environment:
             DEBUG: "FALSE"

dsBase_6.3.4.tar.gz dsBase_6.3.5-permissive.tar.gzdsBase_6.3.4.tar.gz renamed to dsBase_6.3.5-permissive.tar.gz

@@ -58,10 +58,10 @@ schedules:
       - master
     always: true
   - cron: "0 2 * * *"
-    displayName: Nightly build - v6.3.4-dev
+    displayName: Nightly build - v6.3.5-dev
     branches:
       include:
-      - v6.3.4-dev
+      - v6.3.5-dev
     always: true
 
 #########################################################################################
@@ -235,7 +235,7 @@ jobs:
   - bash: |
       R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'https://localhost:8443', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(o)"
 
-      R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='https://localhost:8443/', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = 'v6.3.4-dev'); opal.logout(opal)"
+      R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='https://localhost:8443/', opts = list(ssl_verifyhost=0, ssl_verifypeer=0)); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = 'v6.3.5-dev'); opal.logout(opal)"
 
       sleep 60