From e051e6543eb9457a70dd2621dd8bdd76556f597e Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 26 May 2026 11:59:13 -0400 Subject: [PATCH 01/39] Update CreateTargetCohortTable.sql --- inst/sql/sql_server/CreateTargetCohortTable.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/sql/sql_server/CreateTargetCohortTable.sql b/inst/sql/sql_server/CreateTargetCohortTable.sql index 2b7f33f..040fc1c 100644 --- a/inst/sql/sql_server/CreateTargetCohortTable.sql +++ b/inst/sql/sql_server/CreateTargetCohortTable.sql @@ -14,6 +14,6 @@ char_type VARCHAR(20) DROP TABLE IF EXISTS @characterization_schema.@attrition_table; CREATE TABLE @characterization_schema.@attrition_table( cohort_definition_id BIGINT, -attr_reason VARCHAR(50), +attr_reason VARCHAR(100), n BIGINT ); From 1ba5b75f777a2505e3eaa8dcd7514e38f9d2e8e7 Mon Sep 17 00:00:00 2001 From: jreps Date: Mon, 1 Jun 2026 16:36:17 -0400 Subject: [PATCH 02/39] adding cohort subsetting in package - updated all analyses to use study population rather than target id as this enables subsetting filtering on min prior obs, first in n days, nesting id, age, gender, and date. --- NAMESPACE | 1 + R/CaseSeries.R | 81 ++-- R/CohortGeneration.R | 219 +++++---- R/DechallengeRechallenge.R | 187 +++++--- R/ExportingCsvFiles.R | 216 ++------- R/HelperFunctions.R | 1 - R/LookupCohortSettings.R | 87 ++-- R/RiskFactorAnalysis.R | 90 ++-- R/RunCharacterization.R | 226 ++++++++-- R/StudyPopulation.R | 112 +++++ R/TargetAnalysis.R | 95 ++-- R/TimeToEvent.R | 60 +-- R/ViewShiny.R | 4 +- .../resultsDataModelSpecification.csv | 417 ++++++++++-------- inst/sql/sql_server/CaseCohorts.sql | 16 +- .../sql_server/CreateTargetCohortTable.sql | 37 +- .../sql/sql_server/DechallengeRechallenge.sql | 6 +- inst/sql/sql_server/DropTargetCohortTable.sql | 21 +- inst/sql/sql_server/DropTimeToEvent.sql | 6 - inst/sql/sql_server/NonCaseCohorts.sql | 138 ++++-- .../sql_server/RechallengeFailCaseSeries.sql | 48 +- inst/sql/sql_server/ResultTables.sql | 2 +- inst/sql/sql_server/TargetCohorts.sql | 316 ++++++++++++- inst/sql/sql_server/TargetCounts.sql | 4 - inst/sql/sql_server/TimeToEvent.sql | 59 +-- man/createCaseSeriesSettings.Rd | 18 +- man/createDechallengeRechallengeSettings.Rd | 10 +- man/createRiskFactorSettings.Rd | 17 +- man/createStudyPopulationSettings.Rd | 56 +++ man/createTargetBaselineSettings.Rd | 14 +- man/createTimeToEventSettings.Rd | 10 +- man/runCharacterizationAnalyses.Rd | 8 +- man/viewCharacterization.Rd | 4 +- tests/testthat/test-CohortGeneration.R | 58 ++- tests/testthat/test-ExportingCsvFiles.R | 3 +- tests/testthat/test-StudyPopulation.R | 53 +++ tests/testthat/test-dechallengeRechallenge.R | 173 ++++++-- tests/testthat/test-runCharacterization.R | 43 +- tests/testthat/test-targetAnalysis.R | 84 ++-- 39 files changed, 1917 insertions(+), 1083 deletions(-) create mode 100644 R/StudyPopulation.R delete mode 100644 inst/sql/sql_server/TargetCounts.sql create mode 100644 man/createStudyPopulationSettings.Rd create mode 100644 tests/testthat/test-StudyPopulation.R diff --git a/NAMESPACE b/NAMESPACE index 0d0c6b2..acf4ed9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,6 +12,7 @@ export(createDechallengeRechallengeSettings) export(createDuringCovariateSettings) export(createRiskFactorSettings) export(createSqliteDatabase) +export(createStudyPopulationSettings) export(createTargetBaselineSettings) export(createTimeToEventSettings) export(exampleOmopConnectionDetails) diff --git a/R/CaseSeries.R b/R/CaseSeries.R index 0c6ec96..fef094e 100644 --- a/R/CaseSeries.R +++ b/R/CaseSeries.R @@ -16,10 +16,8 @@ #' Create aggregate covariate study settings #' -#' @param targetIds A list of cohortIds for the target cohorts +#' @param studyPopulationSettings A List of object created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts -#' @param limitToFirstInNDays whether to limit each target cohort to the first entry into the cohort per N days per subject -#' @param minPriorObservation The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index #' @param outcomeWashoutDays Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis #' @template timeAtRisk #' @param caseCovariateSettings An object created using \code{createDuringCovariateSettings} @@ -32,10 +30,12 @@ #' @examples #' #' caseSeriesSetting <- createCaseSeriesSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' minPriorObservation = 365, +#' limitToFirstInNDays = 365 +#' ), #' outcomeIds = c(3), -#' limitToFirstInNDays = 365, -#' minPriorObservation = 365, #' outcomeWashoutDays = 90, #' riskWindowStart = 1, #' startAnchor = "cohort start", @@ -47,10 +47,8 @@ #' #' @export createCaseSeriesSettings <- function( - targetIds, + studyPopulationSettings, outcomeIds, - limitToFirstInNDays = 99999, - minPriorObservation = 0, outcomeWashoutDays = 0, riskWindowStart = 1, startAnchor = "cohort start", @@ -70,11 +68,11 @@ createCaseSeriesSettings <- function( ) { errorMessages <- checkmate::makeAssertCollection() # check targetIds is a vector of int/double - .checkCohortIds( - cohortIds = targetIds, - type = "target", - errorMessages = errorMessages - ) + #.checkCohortIds( + # cohortIds = targetIds, + # type = "target", + # errorMessages = errorMessages + #) # check outcomeIds is a vector of int/double .checkCohortIds( cohortIds = outcomeIds, @@ -95,10 +93,10 @@ createCaseSeriesSettings <- function( ) # check minPriorObservation - .checkMinPriorObservation( - minPriorObservation = minPriorObservation, - errorMessages = errorMessages - ) + #.checkMinPriorObservation( + # minPriorObservation = minPriorObservation, + # errorMessages = errorMessages + #) # add check for outcomeWashoutDays and nlimitToFirstInNDays @@ -125,10 +123,10 @@ createCaseSeriesSettings <- function( checkmate::reportAssertions(errorMessages) # check unique Ts and Os - if (length(targetIds) != length(unique(targetIds))) { - message("targetIds have duplicates - making unique") - targetIds <- unique(targetIds) - } + #if (length(targetIds) != length(unique(targetIds))) { + # message("targetIds have duplicates - making unique") + # targetIds <- unique(targetIds) + #} if (length(outcomeIds) != length(unique(outcomeIds))) { message("outcomeIds have duplicates - making unique") outcomeIds <- unique(outcomeIds) @@ -137,9 +135,7 @@ createCaseSeriesSettings <- function( # create list result <- list( - targetIds = targetIds, - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, + studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), outcomeIds = outcomeIds, outcomeWashoutDays = outcomeWashoutDays, riskWindowStart = riskWindowStart, @@ -168,8 +164,8 @@ computeCaseSeriesAnalyses <- function( characterizationDatabaseSchema, characterizationTable, # contains char cohorts - targetSettingsTable, # contains map between settings and char cohort id caseSettingsTable, # contains map between settings and case id + caseCountTable, # new tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), settings, @@ -180,6 +176,7 @@ computeCaseSeriesAnalyses <- function( minCellCount = 0, progressBar = interactive(), executionId, + minCaseSize = minCaseSize, ...) { if(missing(outputFolder)){ @@ -199,28 +196,20 @@ computeCaseSeriesAnalyses <- function( start <- Sys.time() message("Case series analysis: Finding temp Ids") - targetIds <- lookupTargets( - connection = connection, - lookupDatabaseSchema = characterizationDatabaseSchema, - lookupTableName = targetSettingsTable, - tempEmulationSchema = tempEmulationSchema, - targetIds = paste0(unique(settings$targetIds), collapse = ','), - limitToFirstInNDays = settings$limitToFirstInNDays, - minPriorObservation = settings$minPriorObservation - ) - caseIds <- lookupCases( connection = connection, lookupDatabaseSchema = characterizationDatabaseSchema, lookupTableName = caseSettingsTable, + countTable = caseCountTable, tempEmulationSchema = tempEmulationSchema, - characterizationTargetIds = paste0(unique(targetIds$characterizationTargetId), collapse = ','), + characterizationTargetIds = paste0(unique(settings$characterizationTargetId), collapse = ','), outcomeIds = paste0(unique(settings$outcomeIds), collapse = ','), outcomeWashoutDays = settings$outcomeWashoutDays, startAnchor = settings$startAnchor, riskWindowStart = settings$riskWindowStart, endAnchor = settings$endAnchor, - riskWindowEnd = settings$riskWindowEnd + riskWindowEnd = settings$riskWindowEnd, + minCaseSize = minCaseSize ) completionTime <- Sys.time() - start @@ -342,8 +331,9 @@ computeCaseSeriesAnalyses <- function( snakeCaseToCamelCase = TRUE ) - result$targetSettings <- targetIds - result$caseSettings <- caseIds + # TODO Removed + ##result$targetSettings <- targetIds + ##result$caseSettings <- caseIds completionTime <- Sys.time() - start message(paste0("Case series analysis: Downloading took ", round(completionTime, digits = 1), " ", units(completionTime))) @@ -403,9 +393,7 @@ getCaseSeriesJobs <- function( FUN = function(outcomeId){ data.frame( - targetId = unique(characterizationSettings[[i]]$targetIds), - limitToFirstInNDays = characterizationSettings[[i]]$limitToFirstInNDays, - minPriorObservation = characterizationSettings[[i]]$minPriorObservation, + characterizationTargetId = unique(characterizationSettings[[i]]$characterizationTargetIds), outcomeId = outcomeId, outcomeWashoutDays = unique(characterizationSettings[[i]]$outcomeWashoutDays), @@ -428,10 +416,9 @@ getCaseSeriesJobs <- function( settings <- c() if(nrow(caseSeriesCombinations) > 0 ){ - jobCols <- c("targetId") + jobCols <- c("characterizationTargetId") settingCols <- c( - "limitToFirstInNDays", "minPriorObservation", "outcomeWashoutDays", "riskWindowStart", "startAnchor", "riskWindowEnd", "endAnchor", @@ -469,10 +456,8 @@ getCaseSeriesJobs <- function( functionName = "computeCaseSeriesAnalyses", settings = as.character(ParallelLogger::convertSettingsToJson( list( - targetIds = unique(restrictedData$targetId[ind]), + characterizationTargetIds = unique(restrictedData$characterizationTargetId[ind]), outcomeIds = unique(restrictedData$outcomeId[ind]), - minPriorObservation = unique(restrictedData$minPriorObservation[ind]), - limitToFirstInNDays = unique(restrictedData$limitToFirstInNDays[ind]), outcomeWashoutDays = unique(restrictedData$outcomeWashoutDays[ind]), riskWindowStart = unique(restrictedData$riskWindowStart[ind]), diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 15faf64..a3fbb8a 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -10,6 +10,8 @@ generateCohorts <- function( targetTable, outcomeDatabaseSchema, outcomeTable, + nestingCohortDatabaseSchema, + nestingCohortTable, outputDatabaseSchema = targetDatabaseSchema, outputTable = 'characterization_cohort', cdmDatabaseSchema, @@ -25,9 +27,15 @@ generateCohorts <- function( # tables names characterizationTableWithHash <- paste0(outputTable, '_',settingHash, '_', dbHash) + targetSettingsTableWithHash <- paste0('target_settings', '_',settingHash, '_', dbHash) + targetAttritionTableWithHash <- paste0('target_attrition', '_',settingHash, '_', dbHash) + targetCountTableWithHash <- paste0('target_count', '_',settingHash, '_', dbHash) + caseSettingsTableWithHash <- paste0('case_settings', '_',settingHash, '_', dbHash) - attritionTableWithHash <- paste0('attrition', '_',settingHash, '_', dbHash) + caseAttritionTableWithHash <- paste0('case_attrition', '_',settingHash, '_', dbHash) + caseCountTableWithHash <- paste0('case_count', '_',settingHash, '_', dbHash) + cohortJobs <- getCohortJobs( characterizationSettings, @@ -58,12 +66,13 @@ generateCohorts <- function( # upload settings: # 1) target_settings cohortJobs$targets if(!is.null(cohortJobs$targets)){ + DatabaseConnector::insertTable( connection = connection, databaseSchema = outputDatabaseSchema, tableName = targetSettingsTableWithHash, - dropTableIfExists = TRUE, - createTable = TRUE, + dropTableIfExists = FALSE, + createTable = FALSE, data = cohortJobs$targets, camelCaseToSnakeCase = TRUE, progressBar = progressBar @@ -94,7 +103,10 @@ generateCohorts <- function( tempEmulationSchema = tempEmulationSchema, characterization_schema = outputDatabaseSchema, characterization_table = characterizationTableWithHash, - attrition_table = attritionTableWithHash + target_attrition_table = targetAttritionTableWithHash, + target_count_table = targetCountTableWithHash, + case_attrition_table = caseAttritionTableWithHash, + case_count_table = caseCountTableWithHash ) DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) @@ -113,7 +125,7 @@ generateCohorts <- function( x = tracker ) - } else{ + } else{ # replace with readr read? tracker <- utils::read.csv(file.path(executionPath,'cohort_job_tracker.csv')) } @@ -134,7 +146,10 @@ generateCohorts <- function( tempEmulationSchema = tempEmulationSchema, characterization_schema = outputDatabaseSchema, characterization_table = characterizationTableWithHash, - attrition_table = attritionTableWithHash + target_attrition_table = targetAttritionTableWithHash, + target_count_table = targetCountTableWithHash, + case_attrition_table = caseAttritionTableWithHash, + case_count_table = caseCountTableWithHash ) DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) @@ -161,7 +176,10 @@ generateCohorts <- function( connectionDetails = connectionDetails, cdmDatabaseSchema = cdmDatabaseSchema, characterizationTable = characterizationTableWithHash, - attritionTable = attritionTableWithHash, + targetAttritionTable = targetAttritionTableWithHash, + caseAttritionTable = caseAttritionTableWithHash, + targetCountTable = targetCountTableWithHash, + caseCountTable = caseCountTableWithHash, targetSettingsTable = targetSettingsTableWithHash, caseSettingsTable = caseSettingsTableWithHash, characterizationDatabaseSchema = outputDatabaseSchema, @@ -170,6 +188,8 @@ generateCohorts <- function( targetTable = targetTable, outcomeDatabaseSchema = outcomeDatabaseSchema, outcomeTable = outcomeTable, + nestingCohortDatabaseSchema = nestingCohortDatabaseSchema, + nestingCohortTable = nestingCohortTable, incremental = incremental, mode = mode, @@ -200,7 +220,10 @@ return(list( characterizationTable = characterizationTableWithHash, targetSettingsTable = targetSettingsTableWithHash, caseSettingsTable = caseSettingsTableWithHash, - attritionTable = attritionTableWithHash + targetAttritionTable = targetAttritionTableWithHash, + caseAttritionTable = caseAttritionTableWithHash, + targetCountTable = targetCountTableWithHash, + caseCountTable = caseCountTableWithHash ) ) } @@ -226,60 +249,16 @@ runCohortGenerationInParallel <- function(x){ getCohortJobs <- function( characterizationSettings, mode, - nTargetJobs # not currently used + nTargetJobs ){ message('Extracting cohort jobs') - targets <- c() + targets <- characterizationSettings$characterizationTargetLookup cases <- c() - # Extracting Target Baseline targets - if(!is.null(characterizationSettings$targetBaselineSettings)){ - - tempTargets <- do.call( - what = 'rbind', - args = lapply( - X = characterizationSettings$targetBaselineSettings, - FUN = function(x){ - data.frame( - targetId = x$targetIds, - limitToFirstInNDays = x$limitToFirstInNDays, - minPriorObservation = x$minPriorObservation - ) - } - ) - ) - - targets <- rbind( - targets, - tempTargets - ) - - } - - # Extracting Risk Factor targets and cases if(!is.null(characterizationSettings$riskFactorSettings)){ - tempTargets <- do.call( - what = 'rbind', - args = lapply( - X = characterizationSettings$riskFactorSettings, - FUN = function(x){ - data.frame( - targetId = x$targetIds, - limitToFirstInNDays = x$limitToFirstInNDays, - minPriorObservation = x$minPriorObservation - ) - } - ) - ) - - targets <- rbind( - targets, - tempTargets - ) - tempCases <- do.call( what = 'rbind', args = lapply( @@ -288,19 +267,18 @@ getCohortJobs <- function( do.call( what = 'rbind', lapply( - X = unique(x$targetIds), + X = unique(x$characterizationTargetIds), FUN = function(y){ data.frame( - targetId = y, - limitToFirstInNDays = x$limitToFirstInNDays, - minPriorObservation = x$minPriorObservation, + characterizationTargetId = y, outcomeId = x$outcomeIds, outcomeWashoutDays = x$outcomeWashoutDays, riskWindowStart = x$riskWindowStart, startAnchor = x$startAnchor, riskWindowEnd = x$riskWindowEnd, endAnchor = x$endAnchor, - runtype = 'risk-factor' + riskFactorSettings = TRUE, + caseSeriesSettings = FALSE ) } ) @@ -319,25 +297,6 @@ getCohortJobs <- function( # Extracting Case Series cases if(!is.null(characterizationSettings$caseSeriesSettings)){ - tempTargets <- do.call( - what = 'rbind', - args = lapply( - X = characterizationSettings$caseSeriesSettings, - FUN = function(x){ - data.frame( - targetId = x$targetIds, - limitToFirstInNDays = x$limitToFirstInNDays, - minPriorObservation = x$minPriorObservation - ) - } - ) - ) - - targets <- rbind( - targets, - tempTargets - ) - tempCases <- do.call( what = 'rbind', args = lapply( @@ -346,19 +305,18 @@ getCohortJobs <- function( do.call( what = 'rbind', lapply( - X = unique(x$targetIds), + X = unique(x$characterizationTargetIds), FUN = function(y){ data.frame( - targetId = y, - limitToFirstInNDays = x$limitToFirstInNDays, - minPriorObservation = x$minPriorObservation, + characterizationTargetId = y, outcomeId = x$outcomeIds, outcomeWashoutDays = x$outcomeWashoutDays, riskWindowStart = x$riskWindowStart, startAnchor = x$startAnchor, riskWindowEnd = x$riskWindowEnd, endAnchor = x$endAnchor, - runtype = 'case-series' + riskFactorSettings = FALSE, + caseSeriesSettings = TRUE ) } ) @@ -379,25 +337,28 @@ getCohortJobs <- function( if(!is.null(nrow(targets))){ jobCols <- c("targetId") + settingsCols <- c("limitToFirstInNDays", "minPriorObservation", + "nestingCohortId", "minAge", "maxAge", + "studyStart", "studyEnd", "genderConceptIds") jobSettings <- targets %>% + dplyr::ungroup() %>% dplyr::select(dplyr::all_of(jobCols)) %>% dplyr::distinct() + jobSettings$nTargetJobs <- rep(1:nTargetJobs, ceiling(nrow(jobSettings) / nTargetJobs))[1:nrow(jobSettings)] targets <- merge(targets, jobSettings, by = jobCols) targets <- unique(targets) %>% dplyr::inner_join( y = targets %>% - dplyr::distinct(.data$limitToFirstInNDays, .data$minPriorObservation) %>% - dplyr::arrange(.data$limitToFirstInNDays, .data$minPriorObservation) %>% + dplyr::select(dplyr::all_of(settingsCols)) %>% + dplyr::distinct() %>% + dplyr::arrange(dplyr::pick(dplyr::all_of(settingsCols))) %>% dplyr::mutate( settingId = dplyr::row_number() ), - by = c("limitToFirstInNDays", "minPriorObservation") - ) %>% - dplyr::mutate( - characterizationTargetId = dplyr::row_number()*10 + by = settingsCols ) @@ -407,7 +368,7 @@ getCohortJobs <- function( toi <- targets %>% dplyr::filter(.data$settingId == !!setId) - settingVal <- toi[1,c("limitToFirstInNDays", "minPriorObservation")] + settingVal <- toi[1,settingsCols] for (i in unique(toi$nTargetJobs)) { ind <- toi$nTargetJobs== i @@ -419,7 +380,13 @@ getCohortJobs <- function( settingId = setId, targetIds = unique(toi$targetId[ind]), limitToFirstInNDays = unique(toi$limitToFirstInNDays[ind]), - minPriorObservation = unique(toi$minPriorObservation[ind]) + minPriorObservation = unique(toi$minPriorObservation[ind]), + nestingCohortId = unique(toi$nestingCohortId[ind]), + minAge = unique(toi$minAge[ind]), + maxAge = unique(toi$maxAge[ind]), + studyStart = unique(toi$studyStart[ind]), + studyEnd = unique(toi$studyEnd[ind]), + genderConceptIds = unique(toi$genderConceptIds[ind]) ))), jobId = paste("targets",i, paste0(settingVal, collapse = "_"), sep = "_") )) @@ -433,13 +400,14 @@ getCohortJobs <- function( cases <- unique(cases) %>% dplyr::group_by( - .data$targetId,.data$limitToFirstInNDays, .data$minPriorObservation, + .data$characterizationTargetId, .data$outcomeWashoutDays, .data$outcomeId, .data$riskWindowStart,.data$startAnchor, .data$riskWindowEnd, .data$endAnchor ) %>% dplyr::summarize( - runtype = paste(.data$runtype, collapse = ',') + riskFactorSettings = any(.data$riskFactorSettings), + caseSeriesSettings = any(.data$caseSeriesSettings) ) %>% dplyr::ungroup() %>% dplyr::inner_join( @@ -457,16 +425,21 @@ getCohortJobs <- function( "riskWindowStart", "startAnchor", "riskWindowEnd", "endAnchor") ) %>% - dplyr::inner_join( - y = targets %>% - dplyr::select("characterizationTargetId", "targetId","limitToFirstInNDays", "minPriorObservation", "nTargetJobs"), - by = c("targetId","limitToFirstInNDays", "minPriorObservation") - ) %>% - dplyr::select(-"targetId",-"limitToFirstInNDays", -"minPriorObservation") %>% + dplyr::distinct() %>% dplyr::mutate( characterizationCaseId = dplyr::row_number() ) + # add nTargetJobs using characterizationTargetId + jobCols <- c("characterizationTargetId") + jobSettings <- cases %>% + dplyr::ungroup() %>% + dplyr::select(dplyr::all_of(jobCols)) %>% + dplyr::distinct() + + jobSettings$nTargetJobs <- rep(1:nTargetJobs, ceiling(nrow(jobSettings) / nTargetJobs))[1:nrow(jobSettings)] + cases <- merge(cases, jobSettings, by = jobCols) + message(paste0('Adding ', length(unique(cases$settingId))*length(unique(cases$nTargetJobs)) ,' Case Cohort Jobs containing ', nrow(cases), ' case cohorts')) nNonCase <- 0 @@ -493,8 +466,8 @@ getCohortJobs <- function( startAnchor = unique(coi$startAnchor[ind]), riskWindowEnd = unique(coi$riskWindowEnd[ind]), endAnchor = unique(coi$endAnchor[ind]), - generateRiskFactors = length(grep('risk-factor', unique(coi$runtype[ind]))) > 0 , - generateCaseSeries = length(grep('case-series', unique(coi$runtype[ind]))) > 0 + generateRiskFactors = any(coi$riskFactorSettings[ind]) , + generateCaseSeries = any(coi$caseSeriesSettings[ind]) ) )), jobId = paste("cases",i, paste0(settingVal, collapse = "_"), sep = "_") @@ -502,7 +475,7 @@ getCohortJobs <- function( if(mode != 'Efficient'){ - if(length(grep('risk-factor', unique(coi$runtype[ind]))) > 0){ + if(any(coi$riskFactorSettings[ind])){ nNonCase <- nNonCase + 1 jobs <- rbind(jobs, data.frame( functionName = 'generateNonCases', @@ -551,7 +524,8 @@ generateTargets <- function( connectionDetails, cdmDatabaseSchema, characterizationTable, - attritionTable, + targetAttritionTable, + targetCountTable, targetSettingsTable, characterizationDatabaseSchema, tempEmulationSchema, @@ -559,6 +533,8 @@ generateTargets <- function( targetTable, outcomeDatabaseSchema, outcomeTable, + nestingCohortDatabaseSchema, + nestingCohortTable, progressBar = interactive(), executionPath, settings, @@ -582,16 +558,28 @@ generateTargets <- function( characterization_schema = characterizationDatabaseSchema, characterization_table = characterizationTable, - attrition_table = attritionTable, + target_attrition_table = targetAttritionTable, + target_count_table = targetCountTable, target_settings_schema = characterizationDatabaseSchema, target_settings_table = targetSettingsTable, limit_to_first_in_n_days = settings$limitToFirstInNDays, min_prior_observation = settings$minPriorObservation, + nesting_cohort_id = settings$nestingCohortId, + min_age = settings$minAge, + max_age = settings$maxAge, + gender_concept_ids = settings$genderConceptIds, + study_start = settings$studyStart, + study_end = settings$studyEnd, + cohort_ids = paste0(settings$targetIds, collapse = ','), cohort_schema = targetDatabaseSchema, cohort_table = targetTable, + + nesting_schema = nestingCohortDatabaseSchema, + nesting_table = nestingCohortTable, + cdm_database_schema = cdmDatabaseSchema ) @@ -624,7 +612,8 @@ generateCases <- function( connectionDetails, cdmDatabaseSchema, characterizationTable, - attritionTable, + caseAttritionTable, + caseCountTable, targetSettingsTable, caseSettingsTable, characterizationDatabaseSchema, @@ -658,7 +647,8 @@ generateCases <- function( characterization_schema = characterizationDatabaseSchema, characterization_table = characterizationTable, - attrition_table = attritionTable, + case_attrition_table = caseAttritionTable, + case_count_table = caseCountTable, case_settings_schema = characterizationDatabaseSchema, case_settings_table = caseSettingsTable, @@ -706,7 +696,8 @@ generateNonCases <- function( connectionDetails, cdmDatabaseSchema, characterizationTable, - attritionTable, + caseAttritionTable, + caseCountTable, targetSettingsTable, caseSettingsTable, characterizationDatabaseSchema, @@ -738,7 +729,8 @@ generateNonCases <- function( characterization_schema = characterizationDatabaseSchema, characterization_table = characterizationTable, - attrition_table = attritionTable, + case_attrition_table = caseAttritionTable, + case_count_table = caseCountTable, case_settings_schema = characterizationDatabaseSchema, case_settings_table = caseSettingsTable, @@ -802,7 +794,11 @@ dropCohorts <- function( characterizationTableWithHash <- paste0(outputTable, '_',settingHash, '_', dbHash) targetSettingsTableWithHash <- paste0('target_settings', '_',settingHash, '_', dbHash) caseSettingsTableWithHash <- paste0('case_settings', '_',settingHash, '_', dbHash) - attritionTableWithHash <- paste0('attrition', '_',settingHash, '_', dbHash) + targetAttritionTableWithHash <- paste0('target_attrition', '_',settingHash, '_', dbHash) + caseAttritionTableWithHash <- paste0('case_attrition', '_',settingHash, '_', dbHash) + targetCountTableWithHash <- paste0('target_count', '_',settingHash, '_', dbHash) + caseCountTableWithHash <- paste0('case_count', '_',settingHash, '_', dbHash) + sql <- SqlRender::loadRenderTranslateSql( sqlFilename = 'DropTargetCohortTable.sql', @@ -811,7 +807,10 @@ dropCohorts <- function( tempEmulationSchema = tempEmulationSchema, characterization_schema = outputDatabaseSchema, characterization_table = characterizationTableWithHash, - attrition_table = attritionTableWithHash, + target_attrition_table = targetAttritionTableWithHash, + case_attrition_table = caseAttritionTableWithHash, + target_count_table = targetCountTableWithHash, + case_count_table = caseCountTableWithHash, target_settings_table = targetSettingsTableWithHash, case_settings_table = caseSettingsTableWithHash ) diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index 18e9d04..99a1872 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -16,7 +16,7 @@ #' Create dechallenge rechallenge study settings #' -#' @param targetIds A list of cohortIds for the target cohorts +#' @param studyPopulationSettings An object created using \code{createStudyPopulationSettings} of a list of \code{createStudyPopulationSettings} that specifies cohort inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts #' @param dechallengeStopInterval An integer specifying the how much time to add to the cohort_end when determining whether the event starts during cohort and ends after #' @param dechallengeEvaluationWindow An integer specifying the period of time after the cohort_end when you cannot see an outcome for a dechallenge success @@ -27,24 +27,29 @@ #' #' @examples #' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' #' #' @export createDechallengeRechallengeSettings <- function( - targetIds, + studyPopulationSettings, outcomeIds, dechallengeStopInterval = 30, - dechallengeEvaluationWindow = 30) { + dechallengeEvaluationWindow = 30 + ) { errorMessages <- checkmate::makeAssertCollection() # check targetIds is a vector of int/double - .checkCohortIds( - cohortIds = targetIds, - type = "target", - errorMessages = errorMessages - ) + #.checkCohortIds( + # cohortIds = targetIds, + # type = "target", + # errorMessages = errorMessages + #) # check outcomeIds is a vector of int/double .checkCohortIds( cohortIds = outcomeIds, @@ -78,7 +83,7 @@ createDechallengeRechallengeSettings <- function( # create data.frame with all combinations result <- list( - targetCohortDefinitionIds = targetIds, + studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), outcomeCohortDefinitionIds = outcomeIds, dechallengeStopInterval = dechallengeStopInterval, dechallengeEvaluationWindow = dechallengeEvaluationWindow @@ -92,6 +97,8 @@ createDechallengeRechallengeSettings <- function( #' #' @template ConnectionDetails #' @template TargetOutcomeTables +#' @param characterizationDatabaseSchema ... +#' @param characterizationTable ... #' @template TempEmulationSchema #' @param settings The settings for the timeToEvent study #' @param databaseId An identifier for the database (string) @@ -125,10 +132,12 @@ createDechallengeRechallengeSettings <- function( #' @export computeDechallengeRechallengeAnalyses <- function( connectionDetails = NULL, - targetDatabaseSchema, - targetTable, + targetDatabaseSchema, # not needed + targetTable, # not needed outcomeDatabaseSchema = targetDatabaseSchema, outcomeTable = targetTable, + characterizationDatabaseSchema, # updated + characterizationTable, # updated tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), settings, databaseId = "database 1", @@ -145,8 +154,8 @@ computeDechallengeRechallengeAnalyses <- function( errorMessages <- checkmate::makeAssertCollection() .checkConnectionDetails(connectionDetails, errorMessages) .checkCohortDetails( - cohortDatabaseSchema = targetDatabaseSchema, - cohortTable = targetTable, + cohortDatabaseSchema = characterizationDatabaseSchema, + cohortTable = characterizationTable, type = "target", errorMessages = errorMessages ) @@ -160,10 +169,10 @@ computeDechallengeRechallengeAnalyses <- function( tempEmulationSchema = tempEmulationSchema, errorMessages = errorMessages ) - .checkDechallengeRechallengeSettings( - settings = settings, - errorMessages = errorMessages - ) + #.checkDechallengeRechallengeSettings( + # settings = settings, + # errorMessages = errorMessages + #) valid <- checkmate::reportAssertions( collection = errorMessages @@ -189,12 +198,12 @@ computeDechallengeRechallengeAnalyses <- function( dbms = connection@dbms, tempEmulationSchema = tempEmulationSchema, database_id = databaseId, - target_database_schema = targetDatabaseSchema, - target_table = targetTable, + characterization_database_schema = characterizationDatabaseSchema, # updated + characterization_table = characterizationTable, # updated, outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, - target_ids = paste(settings$targetCohortDefinitionIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), + characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow ) @@ -237,8 +246,8 @@ computeDechallengeRechallengeAnalyses <- function( message( paste0( "Computing dechallenge rechallenge for ", - length(settings$targetCohortDefinitionIds), " target ids and ", - length(settings$outcomeCohortDefinitionIds), " outcome ids took ", + length(settings$characterizationTargetIds), " target ids and ", + length(settings$outcomeIds), " outcome ids took ", signif(delta, 3), " ", attr(delta, "units") ) @@ -260,6 +269,9 @@ computeDechallengeRechallengeAnalyses <- function( #' #' @template ConnectionDetails #' @template TargetOutcomeTables +#' @param characterizationDatabaseSchema ... +#' @param characterizationTable ... +#' @param targetSettingsTable The name of the target settings table #' @template TempEmulationSchema #' @param settings The settings for the timeToEvent study #' @param databaseId An identifier for the database (string) @@ -298,6 +310,9 @@ computeRechallengeFailCaseSeriesAnalyses <- function( targetTable, outcomeDatabaseSchema = targetDatabaseSchema, outcomeTable = targetTable, + characterizationDatabaseSchema, # updated + characterizationTable, # updated + targetSettingsTable, # added tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), settings, databaseId = "database 1", @@ -350,6 +365,9 @@ computeRechallengeFailCaseSeriesAnalyses <- function( DatabaseConnector::disconnect(connection) ) + # TODO: lookup targetIds based on settings$studyPopulationSettings + + message("Computing dechallenge rechallenge fails results") sql <- SqlRender::loadRenderTranslateSql( sqlFilename = "RechallengeFailCaseSeries.sql", @@ -357,12 +375,15 @@ computeRechallengeFailCaseSeriesAnalyses <- function( dbms = connection@dbms, tempEmulationSchema = tempEmulationSchema, database_id = databaseId, + characterization_database_schema = characterizationDatabaseSchema, # updated + characterization_table = characterizationTable, # updated + target_settings = targetSettingsTable, #updated target_database_schema = targetDatabaseSchema, target_table = targetTable, outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, - target_ids = paste(settings$targetCohortDefinitionIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), + characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow, show_subject_id = showSubjectId @@ -406,8 +427,8 @@ computeRechallengeFailCaseSeriesAnalyses <- function( message( paste0( "Computing dechallenge failed case series for ", - length(settings$targetCohortDefinitionIds), " target IDs and ", - length(settings$outcomeCohortDefinitionIds), " outcome IDs took ", + length(settings$characterizationTargetIds), " target IDs and ", + length(settings$outcomeIds), " outcome IDs took ", signif(delta, 3), " ", attr(delta, "units") ) @@ -432,8 +453,8 @@ getDechallengeRechallengeJobs <- function( return(NULL) } ind <- 1:length(characterizationSettings) - targetIds <- lapply(ind, function(i) { - characterizationSettings[[i]]$targetCohortDefinitionIds + characterizationTargetIds <- lapply(ind, function(i) { + characterizationSettings[[i]]$characterizationTargetIds }) outcomeIds <- lapply(ind, function(i) { characterizationSettings[[i]]$outcomeCohortDefinitionIds @@ -451,10 +472,10 @@ getDechallengeRechallengeJobs <- function( what = "rbind", args = lapply( - 1:length(targetIds), + 1:length(characterizationTargetIds), function(i) { result <- expand.grid( - targetId = targetIds[[i]], + characterizationTargetId = characterizationTargetIds[[i]], outcomeId = outcomeIds[[i]] ) result$dechallengeStopInterval <- dechallengeStopIntervals[[i]] @@ -467,7 +488,7 @@ getDechallengeRechallengeJobs <- function( tcount <- nrow( combinations %>% dplyr::count( - .data$targetId, + .data$characterizationTargetId, .data$dechallengeStopInterval, .data$dechallengeEvaluationWindow ) @@ -490,12 +511,12 @@ getDechallengeRechallengeJobs <- function( if (tcount >= ocount) { threadDf <- combinations %>% dplyr::count( - .data$targetId, + .data$characterizationTargetId, .data$dechallengeStopInterval, .data$dechallengeEvaluationWindow ) threadDf$nTargetJobs <- rep(1:nTargetJobs, ceiling(tcount / nTargetJobs))[1:tcount] - mergeColumn <- c("targetId", "dechallengeStopInterval", "dechallengeEvaluationWindow") + mergeColumn <- c("characterizationTargetId", "dechallengeStopInterval", "dechallengeEvaluationWindow") } else { threadDf <- combinations %>% dplyr::count( @@ -508,43 +529,75 @@ getDechallengeRechallengeJobs <- function( } combinations <- merge(combinations, threadDf, by = mergeColumn) - sets <- lapply( - X = 1:max(threadDf$nTargetJobs), - FUN = function(i) { - createDechallengeRechallengeSettings( - targetIds = unique(combinations$targetId[combinations$nTargetJobs == i]), - outcomeIds = unique(combinations$outcomeId[combinations$nTargetJobs == i]), - dechallengeStopInterval = unique(combinations$dechallengeStopInterval[combinations$nTargetJobs == i]), - dechallengeEvaluationWindow = unique(combinations$dechallengeEvaluationWindow[combinations$nTargetJobs == i]) - ) - } - ) + + + # create settings based on dechallengeStopInterval/dechallengeEvaluationWindow + + settingCols <- c("dechallengeStopInterval", "dechallengeEvaluationWindow") + executionSettings <- combinations %>% + dplyr::select(dplyr::all_of(settingCols)) %>% + dplyr::distinct() %>% + dplyr::mutate( + settingId = dplyr::row_number() + ) + combinations <- merge(combinations, executionSettings, by = settingCols) + # recreate settings settings <- c() - for (i in 1:length(sets)) { - settings <- rbind( - settings, - data.frame( - functionName = "computeDechallengeRechallengeAnalyses", - settings = as.character(ParallelLogger::convertSettingsToJson( - sets[[i]] - )), - executionFolder = paste0("dr_", i), - jobId = paste0("dr_", i) + for (settingId in unique(combinations$settingId)) { + for (targetJobId in unique(combinations$nTargetJobs)){ + + restrictedCombo <- combinations %>% + dplyr::filter(.data$settingId == !!settingId) %>% + dplyr::filter(.data$nTargetJobs == !!targetJobId) + + + settings <- rbind( + settings, + data.frame( + functionName = "computeDechallengeRechallengeAnalyses", + settings = as.character(ParallelLogger::convertSettingsToJson( + list( + characterizationTargetIds = unique(restrictedCombo$characterizationTargetId), + outcomeIds = unique(restrictedCombo$outcomeId), + dechallengeStopInterval = unique(restrictedCombo$dechallengeStopInterval), + dechallengeEvaluationWindow = unique(restrictedCombo$dechallengeEvaluationWindow) + ) + )), + executionFolder = paste("dr", targetJobId, + unique(restrictedCombo$dechallengeStopInterval), + unique(restrictedCombo$dechallengeEvaluationWindow), + sep = '_'), + jobId = paste("dr", targetJobId, + unique(restrictedCombo$dechallengeStopInterval), + unique(restrictedCombo$dechallengeEvaluationWindow), + sep = '_') + ) ) - ) - settings <- rbind( - settings, - data.frame( - functionName = "computeRechallengeFailCaseSeriesAnalyses", - settings = as.character(ParallelLogger::convertSettingsToJson( - sets[[i]] - )), - executionFolder = paste0("rfcs_", i), - jobId = paste0("rfcs_", i) + settings <- rbind( + settings, + data.frame( + functionName = "computeRechallengeFailCaseSeriesAnalyses", + settings = as.character(ParallelLogger::convertSettingsToJson( + list( + characterizationTargetIds = unique(restrictedCombo$characterizationTargetId), + outcomeIds = unique(restrictedCombo$outcomeId), + dechallengeStopInterval = unique(restrictedCombo$dechallengeStopInterval), + dechallengeEvaluationWindow = unique(restrictedCombo$dechallengeEvaluationWindow) + ) + )), + executionFolder = paste("rfcs", targetJobId, + unique(restrictedCombo$dechallengeStopInterval), + unique(restrictedCombo$dechallengeEvaluationWindow), + sep = '_'), + jobId = paste("rfcs", targetJobId, + unique(restrictedCombo$dechallengeStopInterval), + unique(restrictedCombo$dechallengeEvaluationWindow), + sep = '_') + ) ) - ) + } } return(settings) diff --git a/R/ExportingCsvFiles.R b/R/ExportingCsvFiles.R index 0e04d11..bf0a094 100644 --- a/R/ExportingCsvFiles.R +++ b/R/ExportingCsvFiles.R @@ -508,208 +508,46 @@ exportAttrition <- function( minCellCount = 0 ){ - # load attrition - if(file.exists(file.path(executionPath, 'attrition', 'result'))){ - andromeda <- Andromeda::loadAndromeda(file.path(executionPath, 'attrition', 'result')) + # export target attrition + if(file.exists(file.path(executionPath, 'target_attrition', 'result'))){ + andromeda <- Andromeda::loadAndromeda(file.path(executionPath, 'target_attrition', 'result')) - # load case series - if(file.exists(file.path(outputFolder, paste0(csvFilePrefix, 'case_settings', '.csv')))){ - andromeda$caseSettings <- utils::read.csv(file.path(outputFolder, paste0(csvFilePrefix, 'case_settings', '.csv'))) - } - - # load targets - if(file.exists(file.path(outputFolder, paste0(csvFilePrefix, 'target_settings', '.csv')))){ - andromeda$targetSettings <- utils::read.csv(file.path(outputFolder, paste0(csvFilePrefix, 'target_settings', '.csv'))) - } - - # if no case or target settings then return - if(is.null(andromeda$caseSettings) & is.null(andromeda$targetSettings)){ - message('No target and/or case setting found but these are required to process attrition') - return(invisible(FALSE)) - } - - # process the attrition into useful numbers with minCellCount + # censor + data <- andromeda$target_attrition %>% dplyr::mutate( + nEvents = ifelse(.data$nEvents < !!minCellCount & .data$nEvents > 0, -1*minCellCount, .data$nEvents), + nPeople = ifelse(.data$nPeople < !!minCellCount & .data$nPeople > 0, -1*minCellCount, .data$nPeople) + ) %>% + dplyr::collect() - if(is.null(andromeda$caseSettings) & !is.null(andromeda$targetSettings)){ - message('Found targets only to do attrition for...') - - targets <- andromeda$attrition %>% - dplyr::inner_join( - y = andromeda$targetSettings %>% - dplyr::mutate( - cohortDefinitionId = .data$characterization_target_id, - databaseId = .data$database_id, - settingId = .data$setting_id - ), - by = c("cohortDefinitionId", "databaseId", "settingId") - ) - - # apply censoring - targets <- targets %>% - dplyr::mutate( - n = ifelse(.data$n < !!minCellCount, -1*minCellCount, .data$n) - ) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - - andromeda$attritionProcessed <- targets - - } - - if(!is.null(andromeda$caseSettings) & !is.null(andromeda$targetSettings)){ - message('Found cases and targets to do attrition for...') - - targets <- andromeda$attrition %>% - dplyr::inner_join( - y = andromeda$targetSettings %>% - dplyr::mutate( - cohortDefinitionId = .data$characterization_target_id, - databaseId = .data$database_id, - settingId = .data$setting_id - ), - by = c("cohortDefinitionId", "databaseId", "settingId") - ) - - cases <- andromeda$attrition %>% - dplyr::inner_join( - andromeda$caseSettings %>% - dplyr::mutate( - cohortDefinitionId = .data$characterization_case_id*10+1, - databaseId = .data$database_id, - settingId = .data$setting_id - ), - by = c("cohortDefinitionId", "databaseId", "settingId") - ) + colnames(data) <- SqlRender::camelCaseToSnakeCase(colnames(data)) - nonCases <- cases %>% - dplyr::mutate( - n_cases = .data$n, - targetDefinitionId = .data$characterization_target_id - ) %>% - dplyr::select("cohortDefinitionId","targetDefinitionId", "databaseId", "settingId", "n_cases") %>% - dplyr::inner_join( - targets %>% - dplyr::mutate( - n_targets = .data$n, - targetDefinitionId = .data$cohortDefinitionId - ) %>% - dplyr::select("targetDefinitionId","databaseId", "settingId", "n_targets"), - by = c("targetDefinitionId", "databaseId", "settingId") - ) %>% - dplyr::left_join( - andromeda$attrition %>% - dplyr::mutate( - cohortDefinitionId = .data$cohortDefinitionId-1, - n_excludes = .data$n - ) %>% - dplyr::select( - "cohortDefinitionId","databaseId", "settingId", "n_excludes" - ), - by = c("cohortDefinitionId","databaseId","settingId") - ) %>% - dplyr::select(-"targetDefinitionId") %>% - dplyr::group_by( - .data$cohortDefinitionId,.data$databaseId, .data$settingId - ) %>% - dplyr::summarise( - n_cases = max(.data$n_cases, na.rm = TRUE), - n_non_cases = max(.data$n_targets, na.rm = TRUE) - sum(.data$n_excludes, na.rm = TRUE), - n_excluded = sum(.data$n_excludes, na.rm = TRUE) + # save the attrition + utils::write.csv( + x = data, + file = file.path(outputFolder, paste0(csvFilePrefix, 'target_attrition', '.csv')), + row.names = FALSE ) + } + # export case attrition + if(file.exists(file.path(executionPath, 'case_attrition', 'result'))){ + andromeda <- Andromeda::loadAndromeda(file.path(executionPath, 'case_attrition', 'result')) - # apply censoring - targets <- targets %>% - dplyr::mutate( - n = ifelse(.data$n < !!minCellCount, -1*minCellCount, .data$n) - ) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - - andromeda$attritionProcessed <- targets - - - cases <- cases %>% - dplyr::mutate( - n = ifelse(.data$n < !!minCellCount, -1*minCellCount, .data$n) - ) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - Andromeda::appendToTable( - tbl = andromeda$attritionProcessed, - data = cases - ) - - nonCasesTemp <- nonCases %>% - dplyr::mutate( - cohortDefinitionId = .data$cohortDefinitionId+1, - attrReason = 'Non-cases', - n = ifelse(.data$n_non_cases < !!minCellCount, -1*minCellCount, .data$n_non_cases) - ) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - - Andromeda::appendToTable( - tbl = andromeda$attritionProcessed, - data = nonCasesTemp - ) - - excluded <- nonCases %>% - dplyr::mutate( - cohortDefinitionId = .data$cohortDefinitionId+1, - attrReason = 'Total excluded', - n = ifelse(.data$n_excluded < !!minCellCount, -1*minCellCount, .data$n_excluded) - ) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - - Andromeda::appendToTable( - tbl = andromeda$attritionProcessed, - data = excluded - ) - - # individual exlcusions that all above minCellCount - - exclusionIds <- excluded %>% - dplyr::select("cohortDefinitionId") %>% - dplyr::pull() - - excludedIndividual <- as.data.frame(andromeda$attrition %>% - dplyr::filter(.data$cohortDefinitionId %in% !!exclusionIds)) - - if(nrow(excludedIndividual) > 0 ){ - - cohortDefinitionIdsToUncensor <- excludedIndividual %>% - dplyr::group_by( - .data$cohortDefinitionId, .data$databaseId, .data$settingId - ) %>% - dplyr::summarise( - uncensor = sum(.data$n > !!minCellCount, na.rm = TRUE) == dplyr::n() - ) %>% - dplyr::filter(.data$uncensor) %>% - dplyr::select("cohortDefinitionId") %>% - dplyr::pull() - - if(length(cohortDefinitionIdsToUncensor) > 0){ - extras <- andromeda$attrition %>% - dplyr::filter(.data$cohortDefinitionId %in% !!cohortDefinitionIdsToUncensor) %>% - dplyr::select("cohortDefinitionId", "attrReason", "n", "databaseId", "settingId") - - Andromeda::appendToTable( - tbl = andromeda$attritionProcessed, - data = extras - ) - } - - } - - } + # censor + data <- andromeda$case_attrition %>% dplyr::mutate( + nEvents = ifelse(.data$nEvents < !!minCellCount & .data$nEvents > 0, -1*minCellCount, .data$nEvents), + nPeople = ifelse(.data$nPeople < !!minCellCount & .data$nPeople > 0, -1*minCellCount, .data$nPeople) + ) %>% + dplyr::collect() - # change the column format - data <- as.data.frame(andromeda$attritionProcessed) colnames(data) <- SqlRender::camelCaseToSnakeCase(colnames(data)) # save the attrition utils::write.csv( x = data, - file = file.path(outputFolder, paste0(csvFilePrefix, 'attrition', '.csv')), + file = file.path(outputFolder, paste0(csvFilePrefix, 'case_attrition', '.csv')), row.names = FALSE - ) + ) } return(invisible(TRUE)) diff --git a/R/HelperFunctions.R b/R/HelperFunctions.R index 16a44d7..a7a0a69 100644 --- a/R/HelperFunctions.R +++ b/R/HelperFunctions.R @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - createExecutionIds <- function(size) { executionIds <- gsub(" ", "", gsub("[[:punct:]]", "", paste(Sys.time(), sample(1000000, size), sep = ""))) return(executionIds) diff --git a/R/LookupCohortSettings.R b/R/LookupCohortSettings.R index 82e42e5..b7640c0 100644 --- a/R/LookupCohortSettings.R +++ b/R/LookupCohortSettings.R @@ -1,55 +1,39 @@ -lookupTargets <- function( - connection, - lookupDatabaseSchema, - lookupTableName, - tempEmulationSchema, - targetIds = NULL, - limitToFirstInNDays = NULL, - minPriorObservation = NULL, - characterizationTargetId = NULL +minSizeCharacterizationIds <- function( + connection, + tempEmulationSchema, + characterizationTargetIds, + minTargetSize, + cohortDatabaseSchema, + targetCountTable ){ - sql <- " - SELECT - characterization_target_id, - target_id, - limit_to_first_in_n_days, - min_prior_observation - - FROM @lookup_schema.@lookup_table lt - {@use_char_id}?{ - WHERE lt.characterization_target_id in (@char_ids); - }:{ - WHERE lt.target_id in (@target_ids) - AND lt.limit_to_first_in_n_days = @limit_to_first_in_n_days - AND lt.min_prior_observation = @min_prior_observation; - } - " + sql <- "SELECT characterization_target_id + FROM @characterization_schema.@target_count_table + WHERE n_people >= @min_target_size + AND characterization_target_id in (@characterization_target_ids); + " sql <- SqlRender::render( sql = sql, - lookup_schema = lookupDatabaseSchema, - lookup_table = lookupTableName, - target_ids = paste0(targetIds, collapse = ','), - limit_to_first_in_n_days = limitToFirstInNDays, - min_prior_observation = minPriorObservation, - use_char_id = !is.null(characterizationTargetId), - char_ids = paste0(characterizationTargetId, collapse = ',') - ) + characterization_schema = cohortDatabaseSchema, + target_count_table = targetCountTable, + min_target_size = minTargetSize, + characterization_target_ids = paste0(characterizationTargetIds, collapse = ',') + ) sql <- SqlRender::translate( sql = sql, targetDialect = attributes(connection)$dbms, tempEmulationSchema = tempEmulationSchema - ) + ) - lookup <- DatabaseConnector::querySql( + res <- DatabaseConnector::querySql( connection = connection, sql = sql, snakeCaseToCamelCase = TRUE - ) + ) - return(lookup) + return(res) } @@ -57,6 +41,7 @@ lookupCases <- function( connection, lookupDatabaseSchema, lookupTableName, + countTable, # new tempEmulationSchema = tempEmulationSchema, characterizationTargetIds, outcomeIds, @@ -64,12 +49,13 @@ lookupCases <- function( startAnchor, riskWindowStart, endAnchor, - riskWindowEnd + riskWindowEnd, + minCaseSize # new ){ sql <- " SELECT - characterization_case_id, + lt.characterization_case_id, characterization_target_id, outcome_id, outcome_washout_days, @@ -79,26 +65,33 @@ lookupCases <- function( risk_window_end FROM @lookup_schema.@lookup_table lt - WHERE characterization_target_id in (@char_ids) - AND outcome_id in (@outcome_ids) - AND outcome_washout_days = @outcome_washout_days - AND start_anchor = '@start_anchor' - AND risk_window_start = @risk_window_start - AND end_anchor = '@end_anchor' - AND risk_window_end = @risk_window_end; + INNER JOIN @lookup_schema.@case_count_table cct + ON lt.characterization_case_id = cct.characterization_case_id + + WHERE lt.characterization_target_id in (@char_ids) + AND lt.outcome_id in (@outcome_ids) + AND lt.outcome_washout_days = @outcome_washout_days + AND lt.start_anchor = '@start_anchor' + AND lt.risk_window_start = @risk_window_start + AND lt.end_anchor = '@end_anchor' + AND lt.risk_window_end = @risk_window_end + AND cct.n_people >= @min_case_size + ; " sql <- SqlRender::render( sql = sql, lookup_schema = lookupDatabaseSchema, lookup_table = lookupTableName, + case_count_table = countTable, char_ids = paste0(characterizationTargetIds, collapse = ','), outcome_ids = paste0(outcomeIds, collapse = ','), outcome_washout_days = outcomeWashoutDays, start_anchor = startAnchor, risk_window_start = riskWindowStart, end_anchor = endAnchor, - risk_window_end = riskWindowEnd + risk_window_end = riskWindowEnd, + min_case_size = minCaseSize ) sql <- SqlRender::translate( diff --git a/R/RiskFactorAnalysis.R b/R/RiskFactorAnalysis.R index 31506f8..841e42b 100644 --- a/R/RiskFactorAnalysis.R +++ b/R/RiskFactorAnalysis.R @@ -16,10 +16,8 @@ #' Create risk factor study settings #' -#' @param targetIds A list of cohortIds for the target cohorts +#' @param studyPopulationSettings A list of objects created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts -#' @param limitToFirstInNDays whether to limit each target cohort to the first entry into the cohort per N days per subject -#' @param minPriorObservation The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index #' @param outcomeWashoutDays Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis #' @template timeAtRisk #' @param covariateSettings An object created using \code{FeatureExtraction::createCovariateSettings} @@ -33,9 +31,12 @@ #' @examples #' #' riskFactorSetting <- createRiskFactorSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' minPriorObservation = 365, +#' limitToFirstInNDays = 99999 +#' ), #' outcomeIds = c(3), -#' minPriorObservation = 365, #' outcomeWashoutDays = 90, #' riskWindowStart = 1, #' startAnchor = "cohort start", @@ -45,11 +46,8 @@ #' #' @export createRiskFactorSettings <- function( - targetIds, + studyPopulationSettings, outcomeIds, - #? indicationIds - limitToFirstInNDays = 99999, - minPriorObservation = 0, outcomeWashoutDays = 0, riskWindowStart = 1, startAnchor = "cohort start", @@ -84,17 +82,15 @@ createRiskFactorSettings <- function( endDays = 0, longTermStartDays = -365, shortTermStartDays = -30 - ), - minTargetSize = 0, - minTwithOSize = 0 + ) ) { errorMessages <- checkmate::makeAssertCollection() # check targetIds is a vector of int/double - .checkCohortIds( - cohortIds = targetIds, - type = "target", - errorMessages = errorMessages - ) + #.checkCohortIds( + # cohortIds = targetIds, + # type = "target", + # errorMessages = errorMessages + #) # check outcomeIds is a vector of int/double .checkCohortIds( cohortIds = outcomeIds, @@ -131,20 +127,20 @@ createRiskFactorSettings <- function( } # check minPriorObservation - .checkMinPriorObservation( - minPriorObservation = minPriorObservation, - errorMessages = errorMessages - ) + #.checkMinPriorObservation( + # minPriorObservation = minPriorObservation, + # errorMessages = errorMessages + #) # add check for outcomeWashoutDays checkmate::reportAssertions(errorMessages) # check unique Ts and Os - if (length(targetIds) != length(unique(targetIds))) { - message("targetIds have duplicates - making unique") - targetIds <- unique(targetIds) - } + #if (length(targetIds) != length(unique(targetIds))) { + # message("targetIds have duplicates - making unique") + # targetIds <- unique(targetIds) + #} if (length(outcomeIds) != length(unique(outcomeIds))) { message("outcomeIds have duplicates - making unique") outcomeIds <- unique(outcomeIds) @@ -153,18 +149,14 @@ createRiskFactorSettings <- function( # create list result <- list( - targetIds = targetIds, - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, + studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), outcomeIds = outcomeIds, outcomeWashoutDays = outcomeWashoutDays, riskWindowStart = riskWindowStart, startAnchor = gsub(' ', '_',startAnchor), riskWindowEnd = riskWindowEnd, endAnchor = gsub(' ', '_',endAnchor), - covariateSettings = covariateSettings, # risk factors - minTargetSize = minTargetSize, - minTwithOSize = minTwithOSize + covariateSettings = covariateSettings # risk factors ) class(result) <- "riskFactorSettings" @@ -188,8 +180,8 @@ computeRiskFactorAnalyses <- function( characterizationDatabaseSchema, characterizationTable, # contains char cohorts - targetSettingsTable, # contains map between settings and char cohort id caseSettingsTable, # contains map between settings and case id + caseCountTable, # new tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), settings, @@ -202,6 +194,7 @@ computeRiskFactorAnalyses <- function( progressBar = interactive(), mode, executionId, + minCaseSize, #new ...) { if(missing(outputFolder)){ @@ -222,28 +215,22 @@ computeRiskFactorAnalyses <- function( start <- Sys.time() message("Risk factor analysis: Finding temp Ids") - targetIds <- lookupTargets( - connection = connection, - lookupDatabaseSchema = characterizationDatabaseSchema, - lookupTableName = targetSettingsTable, - tempEmulationSchema = tempEmulationSchema, - targetIds = paste0(unique(settings$targetIds), collapse = ','), - limitToFirstInNDays = settings$limitToFirstInNDays, - minPriorObservation = settings$minPriorObservation - ) + # TODO update this using settings$studyPopulationSettings caseIds <- lookupCases( connection = connection, lookupDatabaseSchema = characterizationDatabaseSchema, lookupTableName = caseSettingsTable, + countTable = caseCountTable, tempEmulationSchema = tempEmulationSchema, - characterizationTargetIds = paste0(unique(targetIds$characterizationTargetId), collapse = ','), + characterizationTargetIds = paste0(unique(settings$characterizationTargetId), collapse = ','), outcomeIds = paste0(unique(settings$outcomeIds), collapse = ','), outcomeWashoutDays = settings$outcomeWashoutDays, startAnchor = settings$startAnchor, riskWindowStart = settings$riskWindowStart, endAnchor = settings$endAnchor, - riskWindowEnd = settings$riskWindowEnd + riskWindowEnd = settings$riskWindowEnd, + minCaseSize = minCaseSize ) # generate the targets, cases and non-cases ids @@ -387,8 +374,9 @@ computeRiskFactorAnalyses <- function( snakeCaseToCamelCase = TRUE ) - result$targetSettings <- targetIds - result$caseSettings <- caseIds + # TODO - what is this used for? + ##result$targetSettings <- settings$characterizationTargetId + ##result$caseSettings <- caseIds completionTime <- Sys.time() - start message(paste0("Risk factor analysis: Calculating SMD and downloading took ", round(completionTime, digits = 1), " ", units(completionTime))) @@ -449,9 +437,7 @@ getRiskFactorJobs <- function( FUN = function(outcomeId){ data.frame( - targetId = unique(characterizationSettings[[i]]$targetIds), - limitToFirstInNDays = characterizationSettings[[i]]$limitToFirstInNDays, - minPriorObservation = characterizationSettings[[i]]$minPriorObservation, + characterizationTargetId = unique(characterizationSettings[[i]]$characterizationTargetIds), outcomeId = outcomeId, outcomeWashoutDays = unique(characterizationSettings[[i]]$outcomeWashoutDays), @@ -471,9 +457,8 @@ getRiskFactorJobs <- function( settings <- c() if(nrow(riskFactorCombinations) > 0 ){ - jobCols <- c("targetId") + jobCols <- c("characterizationTargetId") settingCols <- c( - "limitToFirstInNDays", "minPriorObservation", "outcomeWashoutDays", "riskWindowStart", "startAnchor", "riskWindowEnd", "endAnchor" @@ -515,11 +500,8 @@ getRiskFactorJobs <- function( functionName = "computeRiskFactorAnalyses", settings = as.character(ParallelLogger::convertSettingsToJson( list( - targetIds = unique(restrictedData$targetId[ind]), + characterizationTargetIds = unique(restrictedData$characterizationTargetId[ind]), outcomeIds = unique(restrictedData$outcomeId[ind]), - minPriorObservation = unique(restrictedData$minPriorObservation[ind]), - limitToFirstInNDays = unique(restrictedData$limitToFirstInNDays[ind]), - outcomeWashoutDays = unique(restrictedData$outcomeWashoutDays[ind]), riskWindowStart = unique(restrictedData$riskWindowStart[ind]), startAnchor = unique(restrictedData$startAnchor[ind]), diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index a41413b..52f0dd2 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -89,11 +89,84 @@ createCharacterizationSettings <- function( caseSeriesSettings = caseSeriesSettings ) + # update the settings replace the popSet with the characterizationTargetIds + settings <- addCharacterizationTargetIds( + settings = settings + ) + class(settings) <- "characterizationSettings" return(settings) } +# this function extracts all the target ids and subset logic then gives each a unique id +# then it replaces the sudyPopulationSettings with characterizationTargetIds +addCharacterizationTargetIds <- function(settings){ + + settingTypes <- c('timeToEventSettings', 'dechallengeRechallengeSettings', + 'targetBaselineSettings', 'riskFactorSettings', 'caseSeriesSettings') + + # extract out studyPopulationSettings + studyPopulationList <- list() + + for(settingType in settingTypes){ + if(!is.null(settings[[settingType]])){ + studyPopulationList <- append( + studyPopulationList, + lapply(settings[[settingType]], function(x){ + x$studyPopulationSettings %>% + dplyr::mutate( + timeToEventSettings = !!settingType == 'timeToEventSettings', + dechallengeRechallengeSettings = !!settingType == 'dechallengeRechallengeSettings', + targetBaselineSettings = !!settingType == 'targetBaselineSettings', + riskFactorSettings = !!settingType == 'riskFactorSettings', + caseSeriesSettings = !!settingType == 'caseSeriesSettings', + ) + }) + ) + } + } + + # get the unique target + subsets + studyPopulation <- unique(do.call('rbind', studyPopulationList)) %>% + dplyr::group_by(dplyr::across(-settingTypes)) %>% + dplyr::summarise( + timeToEventSettings = any(.data$timeToEventSettings), + dechallengeRechallengeSettings = any(.data$dechallengeRechallengeSettings), + targetBaselineSettings = any(.data$targetBaselineSettings), + riskFactorSettings = any(.data$riskFactorSettings), + caseSeriesSettings = any(.data$caseSeriesSettings), + ) + # give a new id called characterizationTargetIds per target and subset + # characterizationTargetId always ends in 0 + studyPopulation$characterizationTargetId <- (1:nrow(studyPopulation))*10 + + settings$characterizationTargetLookup <- studyPopulation + + # Now update the settings to replace studyPopulationSettings with characterizationTargetIds + for(settingType in settingTypes){ + if(!is.null(settings[[settingType]])){ + + for(i in 1:length(settings[[settingType]])){ + + popSet <- settings[[settingType]][[i]]$studyPopulationSettings + settings[[settingType]][[i]]$studyPopulationSettings <- NULL + + studyPopulationInSetting <- merge( + x = popSet, + y = studyPopulation, + by = colnames(popSet) + ) + settings[[settingType]][[i]]$characterizationTargetIds <- unique(studyPopulationInSetting$characterizationTargetId) + + } + + } + }# end updating settings + + return(settings) +} + #' Save the characterization settings as a json #' @description @@ -194,6 +267,8 @@ loadCharacterizationSettings <- function( #' @param connectionDetails The connection details to the database containing the OMOP CDM data #' @template TargetOutcomeTables #' @template TempEmulationSchema +#' @param nestingCohortTable The cohort table to extract the nesting cohort from +#' @param nestingCohortDatabaseSchema The schema containing the nestingCohortTable #' @param outputDatabaseSchema The schema where the characterization cohort table will be saved into #' @param outputTable The table name where the characterization cohort table will be saved into #' @param cdmDatabaseSchema The schema with the OMOP CDM data @@ -212,6 +287,8 @@ loadCharacterizationSettings <- function( #' @param minCovariateCount The minimum number of patients who must have the covariate when running aggregate covariates #' @param mode Select from Efficient (no exclusions to target based on washout)/CohortIncidence (excludes targets with outcome in washout if they have no time at risk)/PatientLevelPrediction (excludes targets with outcome during washout prior to index) #' @param minSMD The minimum standardized mean difference for the risk factor analysis +#' @param minTargetSize The minimum target size to be included in targetBaseline, riskFactor or caseSeries +#' @param minCaseSize The minimum case or non-case size to be included in riskFactor or caseSeries #' @family LargeScale #' #' @return @@ -248,6 +325,8 @@ runCharacterizationAnalyses <- function( targetTable, outcomeDatabaseSchema, outcomeTable, + nestingCohortTable, + nestingCohortDatabaseSchema, outputDatabaseSchema = targetDatabaseSchema, outputTable = 'characterization_cohort', tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), @@ -263,10 +342,12 @@ runCharacterizationAnalyses <- function( threads = 1, cohortGenerationThreads = NULL, nTargetJobs = 1, - minCharacterizationMean = 0.001, # is this global or within cov set? - minCovariateCount = 0, # is this global or within cov set? + minCharacterizationMean = 0.001, + minCovariateCount = 0, mode = 'CohortIncidence', - minSMD = 0 + minSMD = 0, + minTargetSize = 0, + minCaseSize = 0 ) { # inputs checks errorMessages <- checkmate::makeAssertCollection() @@ -426,6 +507,10 @@ runCharacterizationAnalyses <- function( targetTable = targetTable, outcomeDatabaseSchema = outcomeDatabaseSchema, outcomeTable = outcomeTable, + + nestingCohortTable = nestingCohortTable, + nestingCohortDatabaseSchema = nestingCohortDatabaseSchema, + outputDatabaseSchema = outputDatabaseSchema, outputTable = outputTable, cdmDatabaseSchema = cdmDatabaseSchema, @@ -447,7 +532,10 @@ runCharacterizationAnalyses <- function( connectionDetails = connectionDetails, tempEmulationSchema = tempEmulationSchema, outputDatabaseSchema = outputDatabaseSchema, - attritionTable = tableNames$attritionTable, + caseAttritionTable = tableNames$caseAttritionTable, + targetAttritionTable = tableNames$targetAttritionTable, + caseCountTable = tableNames$caseCountTable, + targetCountTable = tableNames$targetCountTable, targetSettingsTable = tableNames$targetSettingsTable, caseSettingsTable = tableNames$caseSettingsTable, dbHash = dbHash, @@ -457,7 +545,7 @@ runCharacterizationAnalyses <- function( minSMD = minSMD ) - # Now loop over the jobs + # Now loop over the analysis jobs inputSettings <- list( connectionDetails = connectionDetails, targetDatabaseSchema = targetDatabaseSchema, @@ -479,6 +567,12 @@ runCharacterizationAnalyses <- function( characterizationTable = tableNames$characterizationTable, targetSettingsTable = tableNames$targetSettingsTable, caseSettingsTable = tableNames$caseSettingsTable, + + targetCountTable = tableNames$targetCountTable, + caseCountTable = tableNames$caseCountTable, + minTargetSize = minTargetSize, + minCaseSize = minCaseSize, + mode = mode, minSMD = minSMD, executionId = executionId @@ -656,7 +750,10 @@ exportSharedObjects <- function( connectionDetails, outputDatabaseSchema, tempEmulationSchema, - attritionTable, + caseAttritionTable, + targetAttritionTable, + caseCountTable, + targetCountTable, targetSettingsTable, caseSettingsTable, @@ -707,15 +804,94 @@ exportSharedObjects <- function( row.names = FALSE ) - # extract attrition, target_settings + + # extract target attrition + # export target attrition table + sql <- SqlRender::render( + sql = "SELECT * FROM @attrition_table;", + attrition_table = paste0(outputDatabaseSchema, '.' ,targetAttritionTable) + ) + sql <- SqlRender::translate( + sql = sql, + targetDialect = attributes(connection)$dbms, + tempEmulationSchema = tempEmulationSchema + ) + + andromeda <- Andromeda::andromeda() + + DatabaseConnector::querySqlToAndromeda( + connection = connection, + sql = sql, + andromeda = andromeda, + andromedaTableName = 'target_attrition', + snakeCaseToCamelCase = TRUE + ) + + addDbAndSettings( + andromeda = andromeda, + databaseId = databaseId, + settingId = executionId + ) + + saveCharacterizationAndromeda( + andromeda = andromeda, + outputFolder = file.path(executionPath, 'target_attrition') + ) + + # export target settings table + sql <- SqlRender::render( + sql = "SELECT * FROM @target_settings_table;", + target_settings_table = paste0(outputDatabaseSchema, '.' ,targetSettingsTable) + ) + sql <- SqlRender::translate( + sql = sql, + targetDialect = attributes(connection)$dbms, + tempEmulationSchema = tempEmulationSchema + ) + data <- DatabaseConnector::querySql( + connection = connection, + sql = sql, + snakeCaseToCamelCase = FALSE + ) + data$database_id <- databaseId + data$setting_id <- executionId + utils::write.csv( + x = formatDouble(data), + file = file.path(saveLocation, paste0(tablePrefix,'target_settings.csv')), + row.names = FALSE + ) + + # export target count table + sql <- SqlRender::render( + sql = "SELECT * FROM @target_count_table;", + target_count_table = paste0(outputDatabaseSchema, '.' ,targetCountTable) + ) + sql <- SqlRender::translate( + sql = sql, + targetDialect = attributes(connection)$dbms, + tempEmulationSchema = tempEmulationSchema + ) + data <- DatabaseConnector::querySql( + connection = connection, + sql = sql, + snakeCaseToCamelCase = FALSE + ) + data$database_id <- databaseId + data$setting_id <- executionId + utils::write.csv( + x = formatDouble(data), + file = file.path(saveLocation, paste0(tablePrefix,'target_counts.csv')), + row.names = FALSE + ) + + # extract case attrition if(!is.null(characterizationSettings$caseSeriesSettings) | - !is.null(characterizationSettings$riskFactorSettings) | - !is.null(characterizationSettings$targetBaselineSettings)){ + !is.null(characterizationSettings$riskFactorSettings) ){ - # export attrition table + # export target attrition table sql <- SqlRender::render( - sql = "SELECT cohort_definition_id, attr_reason, n FROM @attrition_table;", - attrition_table = paste0(outputDatabaseSchema, '.' ,attritionTable) + sql = "SELECT * FROM @attrition_table;", + attrition_table = paste0(outputDatabaseSchema, '.' ,caseAttritionTable) ) sql <- SqlRender::translate( sql = sql, @@ -729,7 +905,7 @@ exportSharedObjects <- function( connection = connection, sql = sql, andromeda = andromeda, - andromedaTableName = 'attrition', + andromedaTableName = 'case_attrition', snakeCaseToCamelCase = TRUE ) @@ -741,13 +917,13 @@ exportSharedObjects <- function( saveCharacterizationAndromeda( andromeda = andromeda, - outputFolder = file.path(executionPath, 'attrition') + outputFolder = file.path(executionPath, 'case_attrition') ) - # export target settings table + # export case settings table sql <- SqlRender::render( - sql = "SELECT target_id, limit_to_first_in_n_days, min_prior_observation, characterization_target_id FROM @target_settings_table;", - target_settings_table = paste0(outputDatabaseSchema, '.' ,targetSettingsTable) + sql = "SELECT characterization_case_id, characterization_target_id, outcome_id, outcome_washout_days, risk_window_start, start_anchor, risk_window_end, end_anchor, risk_factor_settings, case_series_settings FROM @case_settings_table;", + case_settings_table = paste0(outputDatabaseSchema, '.' ,caseSettingsTable) ) sql <- SqlRender::translate( sql = sql, @@ -763,20 +939,14 @@ exportSharedObjects <- function( data$setting_id <- executionId utils::write.csv( x = formatDouble(data), - file = file.path(saveLocation, paste0(tablePrefix,'target_settings.csv')), + file = file.path(saveLocation, paste0(tablePrefix,'case_settings.csv')), row.names = FALSE ) - } - - # extract case_settings - if(!is.null(characterizationSettings$caseSeriesSettings) | - !is.null(characterizationSettings$riskFactorSettings) ){ - - # export target settings table + # export case count table sql <- SqlRender::render( - sql = "SELECT characterization_case_id, characterization_target_id, outcome_id, outcome_washout_days, risk_window_start, start_anchor, risk_window_end, end_anchor, runtype FROM @case_settings_table;", - case_settings_table = paste0(outputDatabaseSchema, '.' ,caseSettingsTable) + sql = "SELECT * FROM @case_count_table;", + case_count_table = paste0(outputDatabaseSchema, '.' ,caseCountTable) ) sql <- SqlRender::translate( sql = sql, @@ -792,7 +962,7 @@ exportSharedObjects <- function( data$setting_id <- executionId utils::write.csv( x = formatDouble(data), - file = file.path(saveLocation, paste0(tablePrefix,'case_settings.csv')), + file = file.path(saveLocation, paste0(tablePrefix,'case_counts.csv')), row.names = FALSE ) diff --git a/R/StudyPopulation.R b/R/StudyPopulation.R new file mode 100644 index 0000000..83aba74 --- /dev/null +++ b/R/StudyPopulation.R @@ -0,0 +1,112 @@ +#' create the study population settings +#' +#' @param targetIds A target cohort id or vector of target cohort ids to do the subsetting to +#' @param limitToFirstInNDays Should only the first exposure in N days per subject be included? +#' @param minPriorObservation The minimum required continuous observation time prior to index +#' date for a person to be included in the cohort. +#' @param nestingCohortId A cohort definition id to restrict the target cohort. Patient in the target cohort +#' are only included if they are also in the nesting cohort at index. +#' @param minAge The minimum age required to be in the target at index +#' @param maxAge The maximum age required to be in the target at index +#' @param studyStartDate The earliest date to be included into the target. Date format is 'yyyymmdd'. +#' @param studyEndDate The latest date to be included into the target. Date format is 'yyyymmdd'. +#' @param genderConceptIds A target cohort subject's gender concept to restrict to +#' +#' +#' @return +#' A data.frame containing all the settings required +#' for creating the study populations of interest +#' @examples +#' # Create study population settings with a washout period of 365 days and +#' restricted to adults for target dates that occur for the first time in 365 days. +#' populationSettings <- createStudyPopulationSettings( +#' targetId = 1, +#' limitToFirstInNDays = 365, +#' minPriorObservation = 365, +#' minAge = 18 +#' ) +#' @export +createStudyPopulationSettings <- function( + targetIds, + limitToFirstInNDays = 0, + minPriorObservation = 0, + nestingCohortId = NULL, + minAge = NULL, + maxAge = NULL, + studyStartDate = NULL, + studyEndDate = NULL, + genderConceptIds = NULL + ) { + + if(!is.null(limitToFirstInNDays)){ + if(!inherits(limitToFirstInNDays, "numeric") & !inherits(limitToFirstInNDays, "integer")){ + stop('minPriorObservation must be numeric') + } + if(limitToFirstInNDays < 0){ + stop('limitToFirstInNDays must be 0 or more') + } + } else{ + stop('limitToFirstInNDays must be a numeric > 0 not NULL') + } + + if(!is.null(minPriorObservation)){ + if(!inherits(minPriorObservation, "numeric") & !inherits(minPriorObservation, "integer")){ + stop('minPriorObservation must be numeric') + } + if(minPriorObservation < 0){ + stop('minPriorObservation must be 0 or more') + } + } else{ + stop('minPriorObservation must be a numeric > 0 not NULL') + } + + if(!is.null(nestingCohortId)){ + if(!inherits(nestingCohortId, "numeric") & !inherits(nestingCohortId, "integer")){ + stop('nestingCohortId must be numeric or NULL') + } + } + + result <- unique(data.frame( + targetId = targetIds, + limitToFirstInNDays = limitToFirstInNDays, + minPriorObservation = minPriorObservation, + nestingCohortId = replaceNull(nestingCohortId, 0), + minAge = replaceNull(minAge,0), + maxAge = replaceNull(maxAge,9999), + studyStart = replaceNull(studyStartDate,''), #'yyyy/mm/dd', + studyEnd = replaceNull(studyEndDate,''), + genderConceptIds = paste0(sort(genderConceptIds), collapse= ',') + )) + + return(result) +} + + +replaceNull <- function(value, nullReplacement){ + if(is.null(value)){ + return(nullReplacement) + } else{ + return(value) + } +} + + +# take a list of studyPopulationSettings and remove redundancy +combineStudyPopulationSettings <- function(studyPopulationSettingslist){ + + if(inherits(studyPopulationSettingslist, "data.frame")){ + studyPopulationSettingslist <- list(studyPopulationSettingslist) + } + + for(i in 1:length(studyPopulationSettingslist)){ + if(!'targetId' %in% colnames(studyPopulationSettingslist[[i]])){ + stop('Incorrect studyPopulationSettingslist') + } + } + + combined <- unique(do.call('rbind', studyPopulationSettingslist)) + + return(combined) +} + + diff --git a/R/TargetAnalysis.R b/R/TargetAnalysis.R index f061ec7..d639d3e 100644 --- a/R/TargetAnalysis.R +++ b/R/TargetAnalysis.R @@ -1,8 +1,6 @@ #' Create target baseline aggregate covariate study settings #' -#' @param targetIds A list of cohortIds for the target cohorts -#' @param limitToFirstInNDays Whether to remove target cohort entries that occur within limitToFirstInNDays of a prior entry. limitToFirstInNDays = 99999 means limit to first entry. -#' @param minPriorObservation The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index +#' @param studyPopulationSettings An object created using \code{createStudyPopulationSettings} or a list of \code{createStudyPopulationSettings} that specifies specific populations of interest #' @param covariateSettings An object created using \code{FeatureExtraction::createCovariateSettings} #' @family Aggregate #' @return @@ -11,16 +9,16 @@ #' @examples #' #' aggregateSetting <- createTargetBaselineSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = 1:2, #' limitToFirstInNDays = 99999, #' minPriorObservation = 365 +#' ) #' ) #' #' @export createTargetBaselineSettings <- function( - targetIds, - limitToFirstInNDays = 99999, - minPriorObservation = 0, + studyPopulationSettings, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE, useDemographicsAge = TRUE, @@ -56,11 +54,11 @@ createTargetBaselineSettings <- function( errorMessages <- checkmate::makeAssertCollection() # check targetIds is a vector of int/double - .checkCohortIds( - cohortIds = targetIds, - type = "target", - errorMessages = errorMessages - ) + #.checkCohortIds( + # cohortIds = targetIds, + # type = "target", + # errorMessages = errorMessages + #) # check covariateSettings .checkCovariateSettings( @@ -78,17 +76,11 @@ createTargetBaselineSettings <- function( stop("Temporal covariateSettings not supported by createAggregateCovariateSettings()") } - # check minPriorObservation - .checkMinPriorObservation( - minPriorObservation = minPriorObservation, - errorMessages = errorMessages - ) + # check studyPopulationSettings # create list result <- list( - targetIds = targetIds, - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, + studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), covariateSettings = covariateSettings ) @@ -105,7 +97,6 @@ computeTargetBaselineAnalyses <- function( targetTable, characterizationDatabaseSchema, characterizationTable, # contains char cohorts - #attritionTable, targetSettingsTable, # contains map between settings and char cohort id tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), settings, @@ -115,6 +106,8 @@ computeTargetBaselineAnalyses <- function( progressBar = interactive(), minCharacterizationMean = 0.01, minCovariateCount = 0, + minTargetSize = 0, # added + targetCountTable, # added executionId, ...) { @@ -131,15 +124,14 @@ computeTargetBaselineAnalyses <- function( DatabaseConnector::disconnect(connection) ) - # first look up the cohort ids for the settings - cohorts <- lookupTargets( + # restrict to cohorts with min size + minSized <- minSizeCharacterizationIds( connection = connection, - lookupDatabaseSchema = characterizationDatabaseSchema, - lookupTableName = targetSettingsTable, tempEmulationSchema = tempEmulationSchema, - targetIds = settings$targetIds, - limitToFirstInNDays = settings$limitToFirstInNDays, - minPriorObservation = settings$minPriorObservation + characterizationTargetIds = settings$characterizationTargetIds, + minTargetSize = minTargetSize, + cohortDatabaseSchema = characterizationDatabaseSchema, + targetCountTable = targetCountTable ) # next run FE on cohortIds @@ -148,8 +140,8 @@ computeTargetBaselineAnalyses <- function( cdmDatabaseSchema = cdmDatabaseSchema, cohortDatabaseSchema = characterizationDatabaseSchema, cohortTable = characterizationTable, - cohortIds = unique(cohorts$characterizationTargetId), - covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettings), + cohortIds = minSized$characterizationTargetId, + covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettingsJson), cdmVersion = cdmVersion, aggregated = TRUE, minCharacterizationMean = minCharacterizationMean, @@ -196,7 +188,7 @@ computeTargetBaselineAnalyses <- function( result$targetCovariatesContinuous <- result$covariatesContinuous result$covariatesContinuous <- NULL - result$targetSettings <- cohorts + ##result$targetSettings <- cohorts # export to andromeda result <- addDbAndSettings( @@ -226,6 +218,8 @@ getTargetBaselineJobs <- function( } ind <- 1:length(characterizationSettings) + # characterizationTargetIds, covariateSettings + # target combinations targetCombinations <- do.call( what = "rbind", @@ -234,9 +228,7 @@ getTargetBaselineJobs <- function( 1:length(characterizationSettings), function(i) { result <- data.frame( - targetIds = unique(characterizationSettings[[i]]$targetIds), - limitToFirstInNDays = characterizationSettings[[i]]$limitToFirstInNDays, - minPriorObservation = characterizationSettings[[i]]$minPriorObservation, + characterizationTargetId = unique(characterizationSettings[[i]]$characterizationTargetIds), covariateSettingsJson = as.character(ParallelLogger::convertSettingsToJson(characterizationSettings[[i]]$covariateSettings)) ) return(result) @@ -246,8 +238,7 @@ getTargetBaselineJobs <- function( settings <- c() if (nrow(targetCombinations) > 0) { - jobCols <- c("targetIds") - settingCols <- c("minPriorObservation", "limitToFirstInNDays") + jobCols <- c("characterizationTargetId") # thread split - assign each target a treat jobSettings <- targetCombinations %>% @@ -256,45 +247,25 @@ getTargetBaselineJobs <- function( jobSettings$nTargetJobs <- rep(1:nTargetJobs, ceiling(nrow(jobSettings) / nTargetJobs))[1:nrow(jobSettings)] targetCombinations <- merge(targetCombinations, jobSettings, by = jobCols) - executionSettings <- targetCombinations %>% - dplyr::select(dplyr::all_of(settingCols)) %>% - dplyr::distinct() %>% - dplyr::mutate( - settingId = dplyr::row_number() - ) - - targetCombinations <- merge(targetCombinations, executionSettings, by = settingCols) - # recreate settings - for (settingId in unique(executionSettings$settingId)) { - settingVal <- executionSettings %>% - dplyr::filter(.data$settingId == !!settingId) %>% - dplyr::select(dplyr::all_of(settingCols)) - - restrictedData <- targetCombinations %>% - dplyr::inner_join(settingVal, by = settingCols) - - for (i in unique(restrictedData$nTargetJobs)) { - ind <- restrictedData$nTargetJobs== i + for (i in unique(targetCombinations$nTargetJobs)) { + ind <- targetCombinations$nTargetJobs== i settings <- rbind( settings, data.frame( functionName = "computeTargetBaselineAnalyses", settings = as.character(ParallelLogger::convertSettingsToJson( list( - targetIds = unique(restrictedData$targetId[ind]), - limitToFirstInNDays = unique(restrictedData$limitToFirstInNDays[ind]), - minPriorObservation = unique(restrictedData$minPriorObservation[ind]), - covariateSettingsJson = combineCovariateSettingsJsons(as.list(restrictedData$covariateSettingsJson[ind])) + characterizationTargetIds = unique(targetCombinations$characterizationTargetId[ind]), + covariateSettingsJson = combineCovariateSettingsJsons(as.list(targetCombinations$covariateSettingsJson[ind])) ) )), - executionFolder = paste("t", i, paste(settingVal, collapse = "_"), sep = "_"), - jobId = paste("t", i, paste(settingVal, collapse = "_"), sep = "_") + executionFolder = paste("t", i, sep = "_"), + jobId = paste("t", i, sep = "_") ) ) } } - } return(settings) } diff --git a/R/TimeToEvent.R b/R/TimeToEvent.R index 4a26613..1ddf39b 100644 --- a/R/TimeToEvent.R +++ b/R/TimeToEvent.R @@ -16,8 +16,9 @@ #' Create time to event study settings #' -#' @param targetIds A list of cohortIds for the target cohorts +#' @param studyPopulationSettings An object created using \code{createStudyPopulationSettings} or a list of \code{createStudyPopulationSettings} that specifies cohort inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts +#' #' @family TimeToEvent #' #' @return @@ -27,23 +28,28 @@ #' # example code #' #' tteSet <- createTimeToEventSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' #' #' @export createTimeToEventSettings <- function( - targetIds, - outcomeIds) { + studyPopulationSettings, + outcomeIds +) { # check indicationIds errorMessages <- checkmate::makeAssertCollection() # check targetIds is a vector of int/double - .checkCohortIds( - cohortIds = targetIds, - type = "target", - errorMessages = errorMessages - ) + #.checkCohortIds( + # cohortIds = targetIds, + # type = "target", + # errorMessages = errorMessages + #) # check outcomeIds is a vector of int/double .checkCohortIds( cohortIds = outcomeIds, @@ -55,7 +61,7 @@ createTimeToEventSettings <- function( # create data.frame with all combinations result <- list( - targetIds = targetIds, + studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), outcomeIds = outcomeIds ) @@ -109,6 +115,8 @@ computeTimeToEventAnalyses <- function( targetTable, outcomeDatabaseSchema = targetDatabaseSchema, outcomeTable = targetTable, + characterizationDatabaseSchema, + characterizationTable, tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), cdmDatabaseSchema, settings, @@ -142,10 +150,10 @@ computeTimeToEventAnalyses <- function( tempEmulationSchema = tempEmulationSchema, errorMessages = errorMessages ) - .checkTimeToEventSettings( - settings = settings, - errorMessages = errorMessages - ) + #.checkTimeToEventSettings( + # settings = settings, + # errorMessages = errorMessages + #) valid <- checkmate::reportAssertions(errorMessages) @@ -163,7 +171,7 @@ computeTimeToEventAnalyses <- function( message("Uploading #cohort_settings") pairs <- expand.grid( - targetCohortDefinitionId = settings$targetIds, + characterizationTargetId = settings$characterizationTargetIds, outcomeCohortDefinitionId = settings$outcomeIds ) @@ -187,8 +195,8 @@ computeTimeToEventAnalyses <- function( tempEmulationSchema = tempEmulationSchema, database_id = databaseId, cdm_database_schema = cdmDatabaseSchema, - target_database_schema = targetDatabaseSchema, - target_table = targetTable, + characterization_schema = characterizationDatabaseSchema, + characterization_table = characterizationTable, outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable ) @@ -263,8 +271,8 @@ getTimeToEventJobs <- function( return(NULL) } ind <- 1:length(characterizationSettings) - targetIds <- lapply(ind, function(i) { - characterizationSettings[[i]]$targetIds + characterizationTargetIds <- lapply(ind, function(i) { + characterizationSettings[[i]]$characterizationTargetIds }) outcomeIds <- lapply(ind, function(i) { characterizationSettings[[i]]$outcomeIds @@ -276,17 +284,17 @@ getTimeToEventJobs <- function( what = "rbind", args = lapply( - 1:length(targetIds), + 1:length(characterizationTargetIds), function(i) { expand.grid( - targetId = targetIds[[i]], + characterizationTargetIds = characterizationTargetIds[[i]], outcomeId = outcomeIds[[i]] ) } ) ) # find out whether more Ts or more Os - tcount <- length(unique(tnos$targetId)) + tcount <- length(unique(tnos$characterizationTargetIds)) ocount <- length(unique(tnos$outcomeId)) if (nTargetJobs > max(tcount, ocount)) { @@ -296,10 +304,10 @@ getTimeToEventJobs <- function( if (tcount >= ocount) { threadDf <- data.frame( - targetId = unique(tnos$targetId), + characterizationTargetIds = unique(tnos$characterizationTargetIds), nTargetJobs = rep(1:nTargetJobs, ceiling(tcount / nTargetJobs))[1:tcount] ) - mergeColumn <- "targetId" + mergeColumn <- "characterizationTargetIds" } else { threadDf <- data.frame( outcomeId = unique(tnos$outcomeId), @@ -312,8 +320,8 @@ getTimeToEventJobs <- function( sets <- lapply( X = 1:max(threadDf$nTargetJobs), FUN = function(i) { - createTimeToEventSettings( - targetIds = unique(tnos$targetId[tnos$nTargetJobs == i]), + list( + characterizationTargetIds = unique(tnos$characterizationTargetIds[tnos$nTargetJobs == i]), outcomeIds = unique(tnos$outcomeId[tnos$nTargetJobs == i]) ) } diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 196cd80..8fea895 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -16,7 +16,9 @@ #' conDet <- exampleOmopConnectionDetails() #' #' tteSet <- createTimeToEventSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2) +#' ), #' outcomeIds = 3 #' ) #' diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 0175359..63c5653 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -1,192 +1,225 @@ -table_name,column_name,data_type,is_required,primary_key,empty_is_na,min_cell_count,description -time_to_event,database_id,varchar(100),Yes,Yes,No,No,The database identifier -time_to_event,target_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort -time_to_event,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort -time_to_event,outcome_type,varchar(100),Yes,Yes,No,No,Is the outvome a first occurrence or repeat -time_to_event,target_outcome_type,varchar(40),Yes,Yes,No,No,When does the outcome occur relative to target -time_to_event,time_to_event,int,Yes,Yes,No,No,The time (in days) from target index to outcome start -time_to_event,num_events,int,Yes,No,No,No,Number of events that occur during the specified time to event -time_to_event,time_scale,varchar(20),Yes,Yes,No,No,time scale for the number of events -rechallenge_fail_case_series,database_id,varchar(100),Yes,Yes,No,No,The database identifier -rechallenge_fail_case_series,dechallenge_stop_interval,int,Yes,Yes,No,No,The time period that É -rechallenge_fail_case_series,dechallenge_evaluation_window,int,Yes,Yes,No,No,The time period that É -rechallenge_fail_case_series,target_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort -rechallenge_fail_case_series,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort -rechallenge_fail_case_series,person_key,int,Yes,Yes,No,No,The dense rank for the patient (an identifier that is not the same as the database) -rechallenge_fail_case_series,subject_id,bigint,No,No,No,No,The person identifier for the failed case series (optional) -rechallenge_fail_case_series,dechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a dechallenge has occurred -rechallenge_fail_case_series,dechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the dechallenge start (number of days after index) -rechallenge_fail_case_series,dechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the dechallenge end (number of days after index) -rechallenge_fail_case_series,dechallenge_outcome_number,int,Yes,Yes,No,No,The number of times an outcome has occurred during the dechallenge -rechallenge_fail_case_series,dechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) -rechallenge_fail_case_series,rechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a rechallenge exposure has occurred -rechallenge_fail_case_series,rechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the rechallenge start (number of days after index) -rechallenge_fail_case_series,rechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the rechallenge end (number of days after index) -rechallenge_fail_case_series,rechallenge_outcome_number,int,Yes,Yes,No,No,The number of times the outcome has occurred during the rechallenge -rechallenge_fail_case_series,rechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) -dechallenge_rechallenge,database_id,varchar(100),Yes,Yes,No,No,The database identifier -dechallenge_rechallenge,dechallenge_stop_interval,int,Yes,Yes,No,No,The dechallenge stop interval -dechallenge_rechallenge,dechallenge_evaluation_window,int,Yes,Yes,No,No,The dechallenge evaluation window -dechallenge_rechallenge,target_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort -dechallenge_rechallenge,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort -dechallenge_rechallenge,num_exposure_eras,int,Yes,No,No,No,The number of exposure eras -dechallenge_rechallenge,num_persons_exposed,int,Yes,No,No,No,The number of persons exposed -dechallenge_rechallenge,num_cases,int,Yes,No,No,No,The number of cases -dechallenge_rechallenge,dechallenge_attempt,int,Yes,No,No,No,The number of dechallenge attempts -dechallenge_rechallenge,dechallenge_fail,int,Yes,No,No,No,The dechallenge fail count -dechallenge_rechallenge,dechallenge_success,int,Yes,No,No,No,The dechallenge success count -dechallenge_rechallenge,rechallenge_attempt,int,Yes,No,No,No,The rechallenge attempt count -dechallenge_rechallenge,rechallenge_fail,int,Yes,No,No,No,The rechallenge fail count -dechallenge_rechallenge,rechallenge_success,int,Yes,No,No,No,The rechallenge success count -dechallenge_rechallenge,pct_dechallenge_attempt,float,Yes,No,No,No,The percentage of dechallenge attempts -dechallenge_rechallenge,pct_dechallenge_success,float,Yes,No,No,No,The percentage of dechallenge success -dechallenge_rechallenge,pct_dechallenge_fail,float,Yes,No,No,No,The percentage of dechallenge fails -dechallenge_rechallenge,pct_rechallenge_attempt,float,Yes,No,No,No,The percentage of rechallenge attempts -dechallenge_rechallenge,pct_rechallenge_success,float,Yes,No,No,No,The percentage of rechallenge success -dechallenge_rechallenge,pct_rechallenge_fail,float,Yes,No,No,No,The percentage of rechallenge fails -analysis_ref,database_id,varchar(100),Yes,Yes,No,No,The database identifier -analysis_ref,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -analysis_ref,analysis_id,int,Yes,Yes,No,No,The analysis identifier -analysis_ref,analysis_name,varchar,Yes,No,No,No,The analysis name -analysis_ref,domain_id,varchar,No,No,No,No,The domain id -analysis_ref,start_day,int,No,No,No,No,The start day -analysis_ref,end_day,int,No,No,No,No,The end day -analysis_ref,is_binary,varchar(1),No,No,No,No,Is this a binary analysis -analysis_ref,missing_means_zero,varchar(1),No,No,No,No,Missing means zero -covariate_ref,database_id,varchar(100),Yes,Yes,No,No,The database identifier -covariate_ref,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -covariate_ref,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier -covariate_ref,covariate_name,varchar,Yes,No,No,No,The covariate name -covariate_ref,analysis_id,int,Yes,No,No,No,The analysis identifier -covariate_ref,concept_id,bigint,Yes,No,No,No,The concept identifier -covariate_ref,value_as_concept_id,int,No,No,No,No,The value as concept_id for features created from observation or measurement values -covariate_ref,collisions,int,No,No,No,No,The number of collisions found for the covariate_id -target_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier -target_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -target_covariates,characterization_target_id,int,Yes,Yes,No,No,The characteriation target id -target_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id -target_covariates,sum_value,int,No,No,No,No,The sum value -target_covariates,average_value,float,No,No,No,No,The average value -target_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier -target_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -target_covariates_continuous,characterization_target_id,int,Yes,Yes,No,No,The characteriation target id -target_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier -target_covariates_continuous,count_value,int,No,No,No,No,The count value -target_covariates_continuous,min_value,float,No,No,No,No,The min value -target_covariates_continuous,max_value,float,No,No,No,No,The max value -target_covariates_continuous,average_value,float,No,No,No,No,The average value -target_covariates_continuous,standard_deviation,float,No,No,No,No,The standard devidation -target_covariates_continuous,median_value,float,No,No,No,No,The median value -target_covariates_continuous,p_10_value,float,No,No,No,No,The 10th percentile -target_covariates_continuous,p_25_value,float,No,No,No,No,The 25th percentile -target_covariates_continuous,p_75_value,float,No,No,No,No,The 75th percentile -target_covariates_continuous,p_90_value,float,No,No,No,No,The 90th percentile -execution_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -execution_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier -execution_settings,database_hash,varchar(50),Yes,No,No,No, -execution_settings,mode,varchar(25),No,No,No,No,Whether Efficient/CohortIncidence/PatientLevelPrediction mode was used for risk factor non-cases -execution_settings,min_characterization_mean,float,No,No,No,No,The minimum fraction of patients who have a covariate for the covariate to be included in results -execution_settings,min_covariate_count,int,No,No,No,No,The minimum number of patients who have a covariate for the covariate to be included in results (useful if cohorts are small) -execution_settings,min_smd,float,No,No,No,No,The minimum standardized mean value a risk factor must have to be included in results -target_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -target_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier -target_settings,characterization_target_id,bigint,Yes,Yes,No,No,The target cohort id after inclusion criteria used internally by characterization -target_settings,target_id,bigint,No,No,No,No,The target cohort id -target_settings,limit_to_first_in_n_days,int,No,No,No,No,Target exposures are only included if they occur >= first_in_n_days days after the last exposure -target_settings,min_prior_observation,int,No,No,No,No,Target exposures with < min_prior_obs days observation before exposure are excluded -case_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -case_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier -case_settings,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" -case_settings,characterization_target_id,bigint,Yes,No,No,No,The target cohort id after inclusion criteria used internally by characterization -case_settings,outcome_id,bigint,No,No,No,No,The outcome cohort id -case_settings,outcome_washout_days,int,No,No,No,No,Outcome exposures with < outcome_washout_days days after the last outcome exposure are excluded -case_settings,start_anchor,varchar(15),No,No,No,No,The start anchor -case_settings,end_anchor,varchar(15),No,No,No,No,The end anchor -case_settings,risk_window_start,int,No,No,No,No,The risk window start -case_settings,risk_window_end,int,No,No,No,No,The risk window end -case_settings,runtype,varchar(50),No,No,No,No,Whether this case was used in risk-factor and/or case-series -case_series_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -case_series_settings,case_pre_target_duration,int,No,No,No,No,The number of days before target index to create the before target period in case series -case_series_settings,case_post_outcome_duration,int,No,No,No,No,The number of days after first outcome after target to create the after outcome period in case series -attrition,cohort_definition_id,bigint,Yes,Yes,No,No,The characterization cohort id -attrition,attr_reason,varchar(100),No,Yes,No,No,Description of cohort or removal -attrition,n,bigint,No,No,No,No,The number of people remaining or removed -attrition,database_id,varchar(100),Yes,Yes,No,No,The database identifier -attrition,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -risk_factor_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier -risk_factor_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -risk_factor_covariates,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" -risk_factor_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id -risk_factor_covariates,non_case_sum_value,int,No,No,No,No,The sum value for the non-cases -risk_factor_covariates,non_case_average_value,float,No,No,No,No,The average value for the non-cases -risk_factor_covariates,case_sum_value,int,No,No,No,No,The sum value of the cases -risk_factor_covariates,case_average_value,float,No,No,No,No,The average value of the cases -risk_factor_covariates,standardized_mean_difference,float,No,No,No,No,The standardized mean difference for the covariate -risk_factor_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier -risk_factor_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -risk_factor_covariates_continuous,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" -risk_factor_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier -risk_factor_covariates_continuous,case_count_value,int,No,No,No,No,The count value -risk_factor_covariates_continuous,case_min_value,float,No,No,No,No,The min value -risk_factor_covariates_continuous,case_max_value,float,No,No,No,No,The max value -risk_factor_covariates_continuous,case_average_value,float,No,No,No,No,The average value -risk_factor_covariates_continuous,case_standard_deviation,float,No,No,No,No,The standard devidation -risk_factor_covariates_continuous,case_median_value,float,No,No,No,No,The median value -risk_factor_covariates_continuous,case_p_10_value,float,No,No,No,No,The 10th percentile -risk_factor_covariates_continuous,case_p_25_value,float,No,No,No,No,The 25th percentile -risk_factor_covariates_continuous,case_p_75_value,float,No,No,No,No,The 75th percentile -risk_factor_covariates_continuous,case_p_90_value,float,No,No,No,No,The 90th percentile -risk_factor_covariates_continuous,non_case_count_value,int,No,No,No,No,The count value -risk_factor_covariates_continuous,non_case_min_value,float,No,No,No,No,The min value -risk_factor_covariates_continuous,non_case_max_value,float,No,No,No,No,The max value -risk_factor_covariates_continuous,non_case_average_value,float,No,No,No,No,The average value -risk_factor_covariates_continuous,non_case_standard_deviation,float,No,No,No,No,The standard devidation -risk_factor_covariates_continuous,non_case_median_value,float,No,No,No,No,The median value -risk_factor_covariates_continuous,non_case_p_10_value,float,No,No,No,No,The 10th percentile -risk_factor_covariates_continuous,non_case_p_25_value,float,No,No,No,No,The 25th percentile -risk_factor_covariates_continuous,non_case_p_75_value,float,No,No,No,No,The 75th percentile -risk_factor_covariates_continuous,non_case_p_90_value,float,No,No,No,No,The 90th percentile -risk_factor_covariates_continuous,standardized_mean_difference,float,No,No,No,No,The standardized mean difference for the covariate -case_series_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier -case_series_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -case_series_covariates,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" -case_series_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id -case_series_covariates,before_sum_value,int,No,No,No,No,The sum value for the non-cases -case_series_covariates,before_average_value,float,No,No,No,No,The average value for the non-cases -case_series_covariates,during_sum_value,int,No,No,No,No,The sum value of the cases -case_series_covariates,during_average_value,float,No,No,No,No,The average value of the cases -case_series_covariates,after_sum_value,int,No,No,No,No,The sum value of the cases -case_series_covariates,after_average_value,float,No,No,No,No,The average value of the cases -case_series_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier -case_series_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -case_series_covariates_continuous,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" -case_series_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier -case_series_covariates_continuous,before_count_value,int,No,No,No,No,The count value -case_series_covariates_continuous,before_min_value,float,No,No,No,No,The min value -case_series_covariates_continuous,before_max_value,float,No,No,No,No,The max value -case_series_covariates_continuous,before_average_value,float,No,No,No,No,The average value -case_series_covariates_continuous,before_standard_deviation,float,No,No,No,No,The standard devidation -case_series_covariates_continuous,before_median_value,float,No,No,No,No,The median value -case_series_covariates_continuous,before_p_10_value,float,No,No,No,No,The 10th percentile -case_series_covariates_continuous,before_p_25_value,float,No,No,No,No,The 25th percentile -case_series_covariates_continuous,before_p_75_value,float,No,No,No,No,The 75th percentile -case_series_covariates_continuous,before_p_90_value,float,No,No,No,No,The 90th percentile -case_series_covariates_continuous,during_min_value,float,No,No,No,No,The min value -case_series_covariates_continuous,during_max_value,float,No,No,No,No,The max value -case_series_covariates_continuous,during_average_value,float,No,No,No,No,The average value -case_series_covariates_continuous,during_standard_deviation,float,No,No,No,No,The standard devidation -case_series_covariates_continuous,during_median_value,float,No,No,No,No,The median value -case_series_covariates_continuous,during_p_10_value,float,No,No,No,No,The 10th percentile -case_series_covariates_continuous,during_p_25_value,float,No,No,No,No,The 25th percentile -case_series_covariates_continuous,during_p_75_value,float,No,No,No,No,The 75th percentile -case_series_covariates_continuous,during_p_90_value,float,No,No,No,No,The 90th percentile -case_series_covariates_continuous,after_count_value,int,No,No,No,No,The count value -case_series_covariates_continuous,after_min_value,float,No,No,No,No,The min value -case_series_covariates_continuous,after_max_value,float,No,No,No,No,The max value -case_series_covariates_continuous,after_average_value,float,No,No,No,No,The average value -case_series_covariates_continuous,after_standard_deviation,float,No,No,No,No,The standard devidation -case_series_covariates_continuous,after_median_value,float,No,No,No,No,The median value -case_series_covariates_continuous,after_p_10_value,float,No,No,No,No,The 10th percentile -case_series_covariates_continuous,after_p_25_value,float,No,No,No,No,The 25th percentile -case_series_covariates_continuous,after_p_75_value,float,No,No,No,No,The 75th percentile -case_series_covariates_continuous,after_p_90_value,float,No,No,No,No,The 90th percentile \ No newline at end of file +table_name,column_name,data_type,is_required,primary_key,empty_is_na,min_cell_count,description +time_to_event,database_id,varchar(100),Yes,Yes,No,No,The database identifier +time_to_event,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort definition id for the target cohort +time_to_event,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort +time_to_event,outcome_type,varchar(100),Yes,Yes,No,No,Is the outvome a first occurrence or repeat +time_to_event,target_outcome_type,varchar(40),Yes,Yes,No,No,When does the outcome occur relative to target +time_to_event,time_to_event,int,Yes,Yes,No,No,The time (in days) from target index to outcome start +time_to_event,num_events,int,Yes,No,No,No,Number of events that occur during the specified time to event +time_to_event,time_scale,varchar(20),Yes,Yes,No,No,time scale for the number of events +rechallenge_fail_case_series,database_id,varchar(100),Yes,Yes,No,No,The database identifier +rechallenge_fail_case_series,dechallenge_stop_interval,int,Yes,Yes,No,No,The time period that É +rechallenge_fail_case_series,dechallenge_evaluation_window,int,Yes,Yes,No,No,The time period that É +rechallenge_fail_case_series,characterization_target_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort +rechallenge_fail_case_series,included,bit,No,No,No,No,Whether this was included in the study population settings eras +rechallenge_fail_case_series,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort +rechallenge_fail_case_series,person_key,int,Yes,Yes,No,No,The dense rank for the patient (an identifier that is not the same as the database) +rechallenge_fail_case_series,subject_id,bigint,No,No,No,No,The person identifier for the failed case series (optional) +rechallenge_fail_case_series,dechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a dechallenge has occurred +rechallenge_fail_case_series,dechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the dechallenge start (number of days after index) +rechallenge_fail_case_series,dechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the dechallenge end (number of days after index) +rechallenge_fail_case_series,dechallenge_outcome_number,int,Yes,Yes,No,No,The number of times an outcome has occurred during the dechallenge +rechallenge_fail_case_series,dechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) +rechallenge_fail_case_series,rechallenge_exposure_number,int,Yes,Yes,No,No,The number of times a rechallenge exposure has occurred +rechallenge_fail_case_series,rechallenge_exposure_start_date_offset,int,Yes,No,No,No,The offset for the rechallenge start (number of days after index) +rechallenge_fail_case_series,rechallenge_exposure_end_date_offset,int,Yes,No,No,No,The offset for the rechallenge end (number of days after index) +rechallenge_fail_case_series,rechallenge_outcome_number,int,Yes,Yes,No,No,The number of times the outcome has occurred during the rechallenge +rechallenge_fail_case_series,rechallenge_outcome_start_date_offset,int,Yes,No,No,No,The offset for the outcome start (number of days after index) +dechallenge_rechallenge,database_id,varchar(100),Yes,Yes,No,No,The database identifier +dechallenge_rechallenge,dechallenge_stop_interval,int,Yes,Yes,No,No,The dechallenge stop interval +dechallenge_rechallenge,dechallenge_evaluation_window,int,Yes,Yes,No,No,The dechallenge evaluation window +dechallenge_rechallenge,characterization_cohort_id,bigint,Yes,Yes,No,No,The characterization cohort definition id for the target cohort +dechallenge_rechallenge,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort +dechallenge_rechallenge,num_exposure_eras,int,Yes,No,No,No,The number of exposure eras +dechallenge_rechallenge,num_persons_exposed,int,Yes,No,No,No,The number of persons exposed +dechallenge_rechallenge,num_cases,int,Yes,No,No,No,The number of cases +dechallenge_rechallenge,dechallenge_attempt,int,Yes,No,No,No,The number of dechallenge attempts +dechallenge_rechallenge,dechallenge_fail,int,Yes,No,No,No,The dechallenge fail count +dechallenge_rechallenge,dechallenge_success,int,Yes,No,No,No,The dechallenge success count +dechallenge_rechallenge,rechallenge_attempt,int,Yes,No,No,No,The rechallenge attempt count +dechallenge_rechallenge,rechallenge_fail,int,Yes,No,No,No,The rechallenge fail count +dechallenge_rechallenge,rechallenge_success,int,Yes,No,No,No,The rechallenge success count +dechallenge_rechallenge,pct_dechallenge_attempt,float,Yes,No,No,No,The percentage of dechallenge attempts +dechallenge_rechallenge,pct_dechallenge_success,float,Yes,No,No,No,The percentage of dechallenge success +dechallenge_rechallenge,pct_dechallenge_fail,float,Yes,No,No,No,The percentage of dechallenge fails +dechallenge_rechallenge,pct_rechallenge_attempt,float,Yes,No,No,No,The percentage of rechallenge attempts +dechallenge_rechallenge,pct_rechallenge_success,float,Yes,No,No,No,The percentage of rechallenge success +dechallenge_rechallenge,pct_rechallenge_fail,float,Yes,No,No,No,The percentage of rechallenge fails +analysis_ref,database_id,varchar(100),Yes,Yes,No,No,The database identifier +analysis_ref,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +analysis_ref,analysis_id,int,Yes,Yes,No,No,The analysis identifier +analysis_ref,analysis_name,varchar,Yes,No,No,No,The analysis name +analysis_ref,domain_id,varchar,No,No,No,No,The domain id +analysis_ref,start_day,int,No,No,No,No,The start day +analysis_ref,end_day,int,No,No,No,No,The end day +analysis_ref,is_binary,varchar(1),No,No,No,No,Is this a binary analysis +analysis_ref,missing_means_zero,varchar(1),No,No,No,No,Missing means zero +covariate_ref,database_id,varchar(100),Yes,Yes,No,No,The database identifier +covariate_ref,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +covariate_ref,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier +covariate_ref,covariate_name,varchar,Yes,No,No,No,The covariate name +covariate_ref,analysis_id,int,Yes,No,No,No,The analysis identifier +covariate_ref,concept_id,bigint,Yes,No,No,No,The concept identifier +covariate_ref,value_as_concept_id,int,No,No,No,No,The value as concept_id for features created from observation or measurement values +covariate_ref,collisions,int,No,No,No,No,The number of collisions found for the covariate_id +target_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier +target_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +target_covariates,characterization_target_id,int,Yes,Yes,No,No,The characteriation target id +target_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id +target_covariates,sum_value,int,No,No,No,No,The sum value +target_covariates,average_value,float,No,No,No,No,The average value +target_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier +target_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +target_covariates_continuous,characterization_target_id,int,Yes,Yes,No,No,The characteriation target id +target_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier +target_covariates_continuous,count_value,int,No,No,No,No,The count value +target_covariates_continuous,min_value,float,No,No,No,No,The min value +target_covariates_continuous,max_value,float,No,No,No,No,The max value +target_covariates_continuous,average_value,float,No,No,No,No,The average value +target_covariates_continuous,standard_deviation,float,No,No,No,No,The standard devidation +target_covariates_continuous,median_value,float,No,No,No,No,The median value +target_covariates_continuous,p_10_value,float,No,No,No,No,The 10th percentile +target_covariates_continuous,p_25_value,float,No,No,No,No,The 25th percentile +target_covariates_continuous,p_75_value,float,No,No,No,No,The 75th percentile +target_covariates_continuous,p_90_value,float,No,No,No,No,The 90th percentile +execution_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +execution_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier +execution_settings,database_hash,varchar(50),Yes,No,No,No, +execution_settings,mode,varchar(25),No,No,No,No,Whether Efficient/CohortIncidence/PatientLevelPrediction mode was used for risk factor non-cases +execution_settings,min_characterization_mean,float,No,No,No,No,The minimum fraction of patients who have a covariate for the covariate to be included in results +execution_settings,min_covariate_count,int,No,No,No,No,The minimum number of patients who have a covariate for the covariate to be included in results (useful if cohorts are small) +execution_settings,min_smd,float,No,No,No,No,The minimum standardized mean value a risk factor must have to be included in results +target_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +target_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier +target_settings,characterization_target_id,bigint,Yes,Yes,No,No,The target cohort id after inclusion criteria used internally by characterization +target_settings,target_id,bigint,No,No,No,No,The target cohort id +target_settings,limit_to_first_in_n_days,int,No,No,No,No,Target exposures are only included if they occur >= first_in_n_days days after the last exposure +target_settings,min_prior_observation,int,No,No,No,No,Target exposures with < min_prior_obs days observation before exposure are excluded +target_settings,nesting_cohort_id,bigint,No,No,No,No,The nesting id for the popualtion of interest +target_settings,min_age,int,No,No,No,No,The min age to be includedfor the popualtion of interest +target_settings,max_age,int,No,No,No,No,The max age to be includedfor the popualtion of interest +target_settings,study_start,date,No,No,No,No,The earliest date to be included for the popualtion of interest +target_settings,study_end,date,No,No,No,No,The latest date to be included for the popualtion of interest +target_settings,gender_concept_ids,varchar(100),No,No,No,No,The gender conept ids to be included for the popualtion of interest +target_settings,time_to_event_settings,bit,No,No,No,No,Whether used in time to event +target_settings,dechallenge_rechallenge_settings,bit,No,No,No,No,Whether used in dechal-rechal +target_settings,target_baseline_settings,bit,No,No,No,No,Whether used in target baseline +target_settings,risk_factor_settings,bit,No,No,No,No,Whether used in risk factor +target_settings,case_series_settings,bit,No,No,No,No,Whether used in case series +case_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier +case_settings,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" +case_settings,characterization_target_id,bigint,Yes,No,No,No,The target cohort id after inclusion criteria used internally by characterization +case_settings,outcome_id,bigint,No,No,No,No,The outcome cohort id +case_settings,outcome_washout_days,int,No,No,No,No,Outcome exposures with < outcome_washout_days days after the last outcome exposure are excluded +case_settings,start_anchor,varchar(15),No,No,No,No,The start anchor +case_settings,end_anchor,varchar(15),No,No,No,No,The end anchor +case_settings,risk_window_start,int,No,No,No,No,The risk window start +case_settings,risk_window_end,int,No,No,No,No,The risk window end +case_settings,risk_factor_settings,varchar(50),No,No,No,No,Whether this case was used in risk-factor +case_settings,case_series_settings,varchar(50),No,No,No,No,Whether this case was used in case-series +case_series_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_series_settings,case_pre_target_duration,int,No,No,No,No,The number of days before target index to create the before target period in case series +case_series_settings,case_post_outcome_duration,int,No,No,No,No,The number of days after first outcome after target to create the after outcome period in case series +risk_factor_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier +risk_factor_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +risk_factor_covariates,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" +risk_factor_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id +risk_factor_covariates,non_case_sum_value,int,No,No,No,No,The sum value for the non-cases +risk_factor_covariates,non_case_average_value,float,No,No,No,No,The average value for the non-cases +risk_factor_covariates,case_sum_value,int,No,No,No,No,The sum value of the cases +risk_factor_covariates,case_average_value,float,No,No,No,No,The average value of the cases +risk_factor_covariates,standardized_mean_difference,float,No,No,No,No,The standardized mean difference for the covariate +risk_factor_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier +risk_factor_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +risk_factor_covariates_continuous,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" +risk_factor_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier +risk_factor_covariates_continuous,case_count_value,int,No,No,No,No,The count value +risk_factor_covariates_continuous,case_min_value,float,No,No,No,No,The min value +risk_factor_covariates_continuous,case_max_value,float,No,No,No,No,The max value +risk_factor_covariates_continuous,case_average_value,float,No,No,No,No,The average value +risk_factor_covariates_continuous,case_standard_deviation,float,No,No,No,No,The standard devidation +risk_factor_covariates_continuous,case_median_value,float,No,No,No,No,The median value +risk_factor_covariates_continuous,case_p_10_value,float,No,No,No,No,The 10th percentile +risk_factor_covariates_continuous,case_p_25_value,float,No,No,No,No,The 25th percentile +risk_factor_covariates_continuous,case_p_75_value,float,No,No,No,No,The 75th percentile +risk_factor_covariates_continuous,case_p_90_value,float,No,No,No,No,The 90th percentile +risk_factor_covariates_continuous,non_case_count_value,int,No,No,No,No,The count value +risk_factor_covariates_continuous,non_case_min_value,float,No,No,No,No,The min value +risk_factor_covariates_continuous,non_case_max_value,float,No,No,No,No,The max value +risk_factor_covariates_continuous,non_case_average_value,float,No,No,No,No,The average value +risk_factor_covariates_continuous,non_case_standard_deviation,float,No,No,No,No,The standard devidation +risk_factor_covariates_continuous,non_case_median_value,float,No,No,No,No,The median value +risk_factor_covariates_continuous,non_case_p_10_value,float,No,No,No,No,The 10th percentile +risk_factor_covariates_continuous,non_case_p_25_value,float,No,No,No,No,The 25th percentile +risk_factor_covariates_continuous,non_case_p_75_value,float,No,No,No,No,The 75th percentile +risk_factor_covariates_continuous,non_case_p_90_value,float,No,No,No,No,The 90th percentile +risk_factor_covariates_continuous,standardized_mean_difference,float,No,No,No,No,The standardized mean difference for the covariate +case_series_covariates,database_id,varchar(100),Yes,Yes,No,No,The database identifier +case_series_covariates,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_series_covariates,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" +case_series_covariates,covariate_id,bigint,Yes,Yes,No,No,The covaraite id +case_series_covariates,before_sum_value,int,No,No,No,No,The sum value for the non-cases +case_series_covariates,before_average_value,float,No,No,No,No,The average value for the non-cases +case_series_covariates,during_sum_value,int,No,No,No,No,The sum value of the cases +case_series_covariates,during_average_value,float,No,No,No,No,The average value of the cases +case_series_covariates,after_sum_value,int,No,No,No,No,The sum value of the cases +case_series_covariates,after_average_value,float,No,No,No,No,The average value of the cases +case_series_covariates_continuous,database_id,varchar(100),Yes,Yes,No,No,The database identifier +case_series_covariates_continuous,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_series_covariates_continuous,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" +case_series_covariates_continuous,covariate_id,bigint,Yes,Yes,No,No,The covariate identifier +case_series_covariates_continuous,before_count_value,int,No,No,No,No,The count value +case_series_covariates_continuous,before_min_value,float,No,No,No,No,The min value +case_series_covariates_continuous,before_max_value,float,No,No,No,No,The max value +case_series_covariates_continuous,before_average_value,float,No,No,No,No,The average value +case_series_covariates_continuous,before_standard_deviation,float,No,No,No,No,The standard devidation +case_series_covariates_continuous,before_median_value,float,No,No,No,No,The median value +case_series_covariates_continuous,before_p_10_value,float,No,No,No,No,The 10th percentile +case_series_covariates_continuous,before_p_25_value,float,No,No,No,No,The 25th percentile +case_series_covariates_continuous,before_p_75_value,float,No,No,No,No,The 75th percentile +case_series_covariates_continuous,before_p_90_value,float,No,No,No,No,The 90th percentile +case_series_covariates_continuous,during_min_value,float,No,No,No,No,The min value +case_series_covariates_continuous,during_max_value,float,No,No,No,No,The max value +case_series_covariates_continuous,during_average_value,float,No,No,No,No,The average value +case_series_covariates_continuous,during_standard_deviation,float,No,No,No,No,The standard devidation +case_series_covariates_continuous,during_median_value,float,No,No,No,No,The median value +case_series_covariates_continuous,during_p_10_value,float,No,No,No,No,The 10th percentile +case_series_covariates_continuous,during_p_25_value,float,No,No,No,No,The 25th percentile +case_series_covariates_continuous,during_p_75_value,float,No,No,No,No,The 75th percentile +case_series_covariates_continuous,during_p_90_value,float,No,No,No,No,The 90th percentile +case_series_covariates_continuous,after_count_value,int,No,No,No,No,The count value +case_series_covariates_continuous,after_min_value,float,No,No,No,No,The min value +case_series_covariates_continuous,after_max_value,float,No,No,No,No,The max value +case_series_covariates_continuous,after_average_value,float,No,No,No,No,The average value +case_series_covariates_continuous,after_standard_deviation,float,No,No,No,No,The standard devidation +case_series_covariates_continuous,after_median_value,float,No,No,No,No,The median value +case_series_covariates_continuous,after_p_10_value,float,No,No,No,No,The 10th percentile +case_series_covariates_continuous,after_p_25_value,float,No,No,No,No,The 25th percentile +case_series_covariates_continuous,after_p_75_value,float,No,No,No,No,The 75th percentile +case_series_covariates_continuous,after_p_90_value,float,No,No,No,No,The 90th percentile +target_counts,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort definition id +target_counts,n_events,bigint,No,No,No,No,The number of events +target_counts,n_people,bigint,No,No,No,No,The number of people +target_counts,database_id,varchar(100),Yes,Yes,No,No,The database identifier +target_counts,setting_id,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_counts,characterization_case_id +case_counts,cohort_type,varchar(50),No,No,No,No,Whether the count is a case or non-case +case_counts,n_events,bigint,No,No,No,No,The number of events +case_counts,n_people,bigint,No,No,No,No,The number of people +case_counts,database_id,varchar(100),Yes,Yes,No,No,The database identifier +case_counts,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +target_attrition,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort id +target_attrition,attr_order,int,No,Yes,No,No,The attrition order +target_attrition,attr_reason,varchar(100),No,No,No,No,Description of removal rule +target_attrition,n_events,bigint,No,No,No,No,The number of events remaining +target_attrition,n_people,bigint,No,No,No,No,The number of people remaining +target_attrition,database_id,varchar(100),Yes,Yes,No,No,The database identifier +target_attrition,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_attrition,characterization_case_id,bigint,Yes,Yes,No,No,The characterization case id +case_attrition,attr_order,int,No,Yes,No,No,The attrition order +case_attrition,attr_reason,varchar(100),No,No,No,No,Description of removal rule +case_attrition,n_events,bigint,No,No,No,No,The number of events remaining +case_attrition,n_people,bigint,No,No,No,No,The number of people remaining +case_attrition,database_id,varchar(100),Yes,Yes,No,No,The database identifier +case_attrition,setting_id,varchar(50),Yes,Yes,No,No,The run identifier diff --git a/inst/sql/sql_server/CaseCohorts.sql b/inst/sql/sql_server/CaseCohorts.sql index b78742f..109bdcc 100644 --- a/inst/sql/sql_server/CaseCohorts.sql +++ b/inst/sql/sql_server/CaseCohorts.sql @@ -131,12 +131,18 @@ FROM #characterization_cases; } --- add to attrition table using risk factor id -INSERT INTO @characterization_schema.@attrition_table +-- add case count table +DELETE FROM @characterization_schema.@case_count_table +WHERE cohort_type = 'Cases' +AND characterization_case_id in +(SELECT DISTINCT cohort_definition_id FROM #characterization_cases); + +INSERT INTO @characterization_schema.@case_count_table SELECT -cohort_definition_id*10+1, -'Cases' as attr_reason, -count(*) as n +cohort_definition_id as characterization_case_id, +'Cases' as cohort_type, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new FROM #characterization_cases diff --git a/inst/sql/sql_server/CreateTargetCohortTable.sql b/inst/sql/sql_server/CreateTargetCohortTable.sql index 2b7f33f..29f584d 100644 --- a/inst/sql/sql_server/CreateTargetCohortTable.sql +++ b/inst/sql/sql_server/CreateTargetCohortTable.sql @@ -11,9 +11,36 @@ observation_period_end_date DATE, char_type VARCHAR(20) ); -DROP TABLE IF EXISTS @characterization_schema.@attrition_table; -CREATE TABLE @characterization_schema.@attrition_table( -cohort_definition_id BIGINT, -attr_reason VARCHAR(50), -n BIGINT +DROP TABLE IF EXISTS @characterization_schema.@target_attrition_table; +CREATE TABLE @characterization_schema.@target_attrition_table( +characterization_target_id BIGINT, +attr_order INT, +attr_reason VARCHAR(200), +n_events BIGINT, +n_people BIGINT +); + +DROP TABLE IF EXISTS @characterization_schema.@case_attrition_table; +CREATE TABLE @characterization_schema.@case_attrition_table( +characterization_case_id BIGINT, +attr_order INT, +attr_reason VARCHAR(200), +n_events BIGINT, +n_people BIGINT +); + +-- count tables +DROP TABLE IF EXISTS @characterization_schema.@target_count_table; +CREATE TABLE @characterization_schema.@target_count_table( +characterization_target_id BIGINT, +n_events BIGINT, +n_people BIGINT +); + +DROP TABLE IF EXISTS @characterization_schema.@case_count_table; +CREATE TABLE @characterization_schema.@case_count_table( +characterization_case_id BIGINT, +cohort_type VARCHAR(10), +n_events BIGINT, +n_people BIGINT ); diff --git a/inst/sql/sql_server/DechallengeRechallenge.sql b/inst/sql/sql_server/DechallengeRechallenge.sql index 8e7f2b7..0f9e106 100644 --- a/inst/sql/sql_server/DechallengeRechallenge.sql +++ b/inst/sql/sql_server/DechallengeRechallenge.sql @@ -1,7 +1,7 @@ IF OBJECT_ID('tempdb..#target_cohort', 'U') IS NOT NULL DROP TABLE #target_cohort; select * into #target_cohort -from @target_database_schema.@target_table -where cohort_definition_id in (@target_ids) +from @characterization_database_schema.@characterization_table +where cohort_definition_id in (@characterization_target_ids) ; IF OBJECT_ID('tempdb..#outcome_cohort', 'U') IS NOT NULL DROP TABLE #outcome_cohort; @@ -15,7 +15,7 @@ select '@database_id' as database_id, @dechallenge_stop_interval as dechallenge_stop_interval, @dechallenge_evaluation_window as dechallenge_evaluation_window, -target_cohort_definition_id, +target_cohort_definition_id as characterization_target_id, -- renamed outcome_cohort_definition_id, num_exposure_eras, num_persons_exposed, diff --git a/inst/sql/sql_server/DropTargetCohortTable.sql b/inst/sql/sql_server/DropTargetCohortTable.sql index 1797b99..2dee2c5 100644 --- a/inst/sql/sql_server/DropTargetCohortTable.sql +++ b/inst/sql/sql_server/DropTargetCohortTable.sql @@ -1,5 +1,24 @@ +{DEFAULT @drop_char_cohorts = true} +{DEFAULT @drop_char_counts = true} +{DEFAULT @drop_char_attr = true} +{DEFAULT @drop_char_settings = true} + +{@drop_char_cohorts}?{ DROP TABLE IF EXISTS @characterization_schema.@characterization_table; -DROP TABLE IF EXISTS @characterization_schema.@attrition_table; +} + +{@drop_char_counts}?{ +DROP TABLE IF EXISTS @characterization_schema.@target_count_table; +DROP TABLE IF EXISTS @characterization_schema.@case_count_table; +} + +{@drop_char_attr}?{ +DROP TABLE IF EXISTS @characterization_schema.@target_attrition_table; +DROP TABLE IF EXISTS @characterization_schema.@case_attrition_table; +} + +{@drop_char_settings}?{ DROP TABLE IF EXISTS @characterization_schema.@target_settings_table; DROP TABLE IF EXISTS @characterization_schema.@case_settings_table; +} diff --git a/inst/sql/sql_server/DropTimeToEvent.sql b/inst/sql/sql_server/DropTimeToEvent.sql index 2a2df0c..afae8b8 100644 --- a/inst/sql/sql_server/DropTimeToEvent.sql +++ b/inst/sql/sql_server/DropTimeToEvent.sql @@ -15,12 +15,6 @@ DROP TABLE #target_w_outcome; TRUNCATE TABLE #two_fu_bounds; DROP TABLE #two_fu_bounds; -TRUNCATE TABLE #t_prior_obs; -DROP TABLE #t_prior_obs; - -TRUNCATE TABLE #t_post_obs; -DROP TABLE #t_post_obs; - TRUNCATE TABLE #two_tte; DROP TABLE #two_tte; diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index b26721a..f555997 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -1,5 +1,8 @@ -- clean this table at the end IF OBJECT_ID('tempdb..#temp_non_cases', 'U') IS NOT NULL DROP TABLE #temp_non_cases; +IF OBJECT_ID('tempdb..#temp_non_cases_with_tar', 'U') IS NOT NULL DROP TABLE #temp_non_cases_with_tar; +IF OBJECT_ID('tempdb..#temp_non_cases_pass_washout', 'U') IS NOT NULL DROP TABLE #temp_non_cases_pass_washout; + SELECT case_settings.characterization_case_id*10+2 as cohort_definition_id, @@ -103,45 +106,130 @@ AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_non ; +-- add counts +-- add case count table +DELETE FROM @characterization_schema.@case_count_table +WHERE cohort_type = 'non-cases' +AND characterization_case_id in +(SELECT DISTINCT (cohort_definition_id-2.0)/10.0 FROM #temp_non_cases); -INSERT INTO @characterization_schema.@attrition_table - +INSERT INTO @characterization_schema.@case_count_table SELECT -cohort_definition_id, -attr_reason, -count(*) as n +(cohort_definition_id-2.0)/10.0 as characterization_case_id, +'non-cases' as cohort_type, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new + +FROM #temp_non_cases temp + +-- not a case +WHERE temp.outcome_during_tar = 0 + + {@use_plp}?{ -- exclude anyone with with outcome during washout before TAR or no TAR + AND temp.outcome_in_washout_before_tar = 0 + AND temp.no_tar_ignoring_outcome_washout = 0 + AND temp.no_tar_obs = 0 + } + {@use_ci}?{ -- exclude anyone without 1+ days of TAR + AND temp.no_tar_ignoring_outcome_washout = 0 + AND temp.no_tar_because_outcome_washout = 0 + AND temp.no_tar_washout_and_obs = 0 + AND temp.no_tar_obs = 0 + } + +GROUP BY +cohort_definition_id +; + -FROM -(SELECT - temp.cohort_definition_id, - temp.row_number, + +-- add the reasons for lost people due to tar/washout +SELECT * +INTO #temp_non_cases_with_tar +FROM #temp_non_cases temp {@use_plp}?{ -CASE -WHEN temp.no_tar_ignoring_outcome_washout = 1 OR temp.no_tar_obs = 1 THEN '1. No TAR due to TAR start > TAR end or observation end' -WHEN temp.outcome_in_washout_before_tar = 1 THEN '2. Outcome occurs during washout' -WHEN temp.outcome_during_tar = 1 THEN '3. Has outcome during TAR' -END attr_reason +WHERE temp.no_tar_ignoring_outcome_washout = 0 AND temp.no_tar_obs = 0 } {@use_ci}?{ -CASE -WHEN temp.no_tar_ignoring_outcome_washout = 1 OR temp.no_tar_obs = 1 THEN '1. No TAR due to TAR start > TAR end or observation end' -WHEN temp.no_tar_because_outcome_washout = 1 OR temp.no_tar_washout_and_obs = 1 THEN '2. No TAR due to outcome washout' -WHEN temp.outcome_during_tar = 1 THEN '3. Has outcome during TAR' -END attr_reason +WHERE temp.no_tar_ignoring_outcome_washout = 0 AND temp.no_tar_obs = 0 } +; -FROM #temp_non_cases temp -) attrition +SELECT * +INTO #temp_non_cases_pass_washout +FROM #temp_non_cases_with_tar temp -WHERE attr_reason IS NOT NULL +{@use_plp}?{ +WHERE temp.outcome_in_washout_before_tar = 0 +} +{@use_ci}?{ +WHERE temp.no_tar_because_outcome_washout = 0 AND temp.no_tar_washout_and_obs = 0 +} +; + + +-- next +DELETE FROM @characterization_schema.@case_attrition_table +WHERE characterization_case_id in +(SELECT DISTINCT (cohort_definition_id-2.0)/10.0 FROM #temp_non_cases); + + +INSERT INTO @characterization_schema.@case_attrition_table + +SELECT +(cohort_definition_id - 2.0)/10.0 as characterization_case_id, +8 as attr_order, -- new +'Has some TAR' as attr_reason, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new + +FROM #temp_non_cases_with_tar +GROUP BY cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +(cohort_definition_id - 2.0)/10.0 as characterization_case_id, +8 as attr_order, -- new +'Has some TAR' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_non_cases +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_non_cases_with_tar) -GROUP BY -cohort_definition_id, -attr_reason ; +INSERT INTO @characterization_schema.@case_attrition_table +SELECT +(cohort_definition_id - 2.0)/10.0 as characterization_case_id, +9 as attr_order, -- new +'Remains after outcome washout' as attr_reason, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new + +FROM #temp_non_cases_pass_washout +GROUP BY cohort_definition_id + +UNION + +SELECT DISTINCT +(cohort_definition_id - 2.0)/10.0 as characterization_case_id, +9 as attr_order, -- new +'Remains after outcome washout' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_non_cases_with_tar +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_non_cases_pass_washout) +; + + + -- cleaning table IF OBJECT_ID('tempdb..#temp_non_cases', 'U') IS NOT NULL DROP TABLE #temp_non_cases; +IF OBJECT_ID('tempdb..#temp_non_cases_with_tar', 'U') IS NOT NULL DROP TABLE #temp_non_cases_with_tar; +IF OBJECT_ID('tempdb..#temp_non_cases_pass_washout', 'U') IS NOT NULL DROP TABLE #temp_non_cases_pass_washout; diff --git a/inst/sql/sql_server/RechallengeFailCaseSeries.sql b/inst/sql/sql_server/RechallengeFailCaseSeries.sql index 02f7b03..a80a699 100644 --- a/inst/sql/sql_server/RechallengeFailCaseSeries.sql +++ b/inst/sql/sql_server/RechallengeFailCaseSeries.sql @@ -1,8 +1,35 @@ +-- here we join the original target table to the subset target table via +-- the target_settings table to class each era as included in the subset or not +-- this ensures to target era id in the final results accounts for all exposures +-- not just those that occur in the subset but we know which exposures occured in +-- the subset (included = 1) IF OBJECT_ID('tempdb..#target_cohort', 'U') IS NOT NULL DROP TABLE #target_cohort; -select * into #target_cohort -from @target_database_schema.@target_table -where cohort_definition_id in (@target_ids) + +SELECT +ts.characterization_cohort_id as cohort_definition_id, +tc.subject_id, +tc.cohort_start_date, +tc.cohort_end_date, +CASE WHEN sc.subject_id is NULL THEN 0 ELSE 1 END included +INTO #target_cohort +FROM @target_database_schema.@target_table tc +INNER JOIN @characterization_schema.@target_settings ts +ON tc.cohort_definition_id = ts.target_id +LEFT JOIN @characterization_schema.@characterization_table sc +ON sc.subject_id = tc.subject_id +AND sc.cohort_start_date = tc.cohort_start_date +AND sc.cohort_definition_id = ts.characterization_target_id +WHERE sc.cohort_definition_id in (@characterization_target_ids) +--where cohort_definition_id in (@target_ids) ; + + +--IF OBJECT_ID('tempdb..#target_cohort', 'U') IS NOT NULL DROP TABLE #target_cohort; +--select * into #target_cohort +--from @target_database_schema.@target_table +--where cohort_definition_id in (@target_ids) +--; + IF OBJECT_ID('tempdb..#outcome_cohort', 'U') IS NOT NULL DROP TABLE #outcome_cohort; select * into #outcome_cohort from @outcome_database_schema.@outcome_table @@ -16,7 +43,8 @@ select '@database_id' as database_id, @dechallenge_stop_interval as dechallenge_stop_interval, @dechallenge_evaluation_window as dechallenge_evaluation_window, - dc1.cohort_definition_id as target_cohort_definition_id, + dc1.cohort_definition_id as characterization_target_id, -- renamed this + dc1.included, -- new: whether the target era is in the target eras subset of interest io1.cohort_definition_id as outcome_cohort_definition_id, dense_rank() over (partition by dc1.cohort_definition_id, io1.cohort_definition_id order by datediff(day, dc0.cohort_start_date, dc1.cohort_start_date), dc1.subject_id) as person_key, {@show_subject_id}?{dc1.subject_id}:{CAST(NULL AS BIGINT) as subject_id}, --this is the field that we would want to allow parameter to make nullable or not export @@ -33,13 +61,15 @@ select into #fail_case_series -from (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) dc0 +from + (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) dc0 inner join (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) dc1 on dc0.subject_id = dc1.subject_id and dc0.cohort_definition_id = dc1.cohort_definition_id and dc0.era_number = 1 - inner join (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) io1 + inner join + (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) io1 on dc1.subject_id = io1.subject_id and io1.cohort_start_date > dc1.cohort_start_date and io1.cohort_start_date <= dc1.cohort_end_date and dc1.cohort_end_date <= dateadd(day,@dechallenge_stop_interval,io1.cohort_start_date) -- exposure ends shortly after outcome starts @@ -48,11 +78,13 @@ from (select *, row_number() over (partition by cohort_definition_id, subject_id and io1.cohort_definition_id = ro0.cohort_definition_id and ro0.cohort_start_date > dc1.cohort_end_date and ro0.cohort_start_date <= dateadd(day,@dechallenge_evaluation_window,dc1.cohort_end_date) --this should be parameterized to be the dechallenge window required for success/failure - inner join (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) de1 + inner join + (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) de1 on dc1.subject_id = de1.subject_id and dc1.cohort_definition_id = de1.cohort_definition_id and de1.cohort_start_date > dateadd(day,@dechallenge_evaluation_window,dc1.cohort_end_date) --using same dechallenge window to detrmine when rechallenge attempt can start - inner join (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) ro1 + inner join + (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) ro1 on de1.subject_id = ro1.subject_id and io1.cohort_definition_id = ro1.cohort_definition_id and ro1.cohort_start_date > de1.cohort_start_date diff --git a/inst/sql/sql_server/ResultTables.sql b/inst/sql/sql_server/ResultTables.sql index 79a13c2..2688468 100644 --- a/inst/sql/sql_server/ResultTables.sql +++ b/inst/sql/sql_server/ResultTables.sql @@ -243,7 +243,7 @@ CREATE TABLE @my_schema.@table_prefixcase_settings ( end_anchor VARCHAR(15), risk_window_start INT, risk_window_end INT, - runtype VARCHAR(50), + runtype VARCHAR(50), -- need to add migration to add risk_factor_settings and case_series_settings PRIMARY KEY (setting_id, database_id,characterization_case_id) ); diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index e687122..e0b1371 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -1,7 +1,14 @@ -- first entry in washout days and min prior obs IF OBJECT_ID('tempdb..#temp_target', 'U') IS NOT NULL DROP TABLE #temp_target; +IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_fist; +IF OBJECT_ID('tempdb..#temp_target_prior', 'U') IS NOT NULL DROP TABLE #temp_target_prior; +IF OBJECT_ID('tempdb..#temp_target_nest', 'U') IS NOT NULL DROP TABLE #temp_target_nest; +IF OBJECT_ID('tempdb..#temp_target_age', 'U') IS NOT NULL DROP TABLE #temp_target_age; +IF OBJECT_ID('tempdb..#temp_target_gender', 'U') IS NOT NULL DROP TABLE #temp_target_gender; +IF OBJECT_ID('tempdb..#temp_target_date', 'U') IS NOT NULL DROP TABLE #temp_target_date; +-- ========================= SELECT CAST(target_settings.characterization_target_id AS BIGINT) AS cohort_definition_id, row_number() over(PARTITION BY CAST(target_settings.characterization_target_id AS BIGINT) ORDER BY temp_cohort.subject_id, temp_cohort.cohort_start_date ASC) AS row_number, @@ -10,14 +17,15 @@ temp_cohort.cohort_start_date, temp_cohort.cohort_end_date, op.observation_period_start_date, op.observation_period_end_date, +temp_cohort.time_between, 'target' as char_type INTO #temp_target FROM (SELECT cohort_definition_id, - @limit_to_first_in_n_days AS limit_to_first_in_n_days, - @min_prior_observation AS min_prior_observation, + --@limit_to_first_in_n_days AS limit_to_first_in_n_days, + --@min_prior_observation AS min_prior_observation, subject_id, cohort_start_date, cohort_end_date, @@ -30,20 +38,113 @@ ON op.person_id = temp_cohort.subject_id AND temp_cohort.cohort_start_date >= op.observation_period_start_date AND temp_cohort.cohort_start_date <= op.observation_period_end_date +-- this is just to get the characterization_target_id INNER JOIN (SELECT * FROM @target_settings_schema.@target_settings_table WHERE limit_to_first_in_n_days = @limit_to_first_in_n_days AND min_prior_observation = @min_prior_observation + -- added: + AND min_age = @min_age + AND max_age = @max_age + AND study_start = '@study_start' + AND study_end = '@study_end' + AND gender_concept_ids = '@gender_concept_ids' + AND nesting_cohort_id = @nesting_cohort_id + + -- add target_ids? + --AND target_id IN (@cohort_ids) + ) target_settings -ON temp_cohort.cohort_definition_id = target_settings.target_id +ON temp_cohort.cohort_definition_id = target_settings.target_id; + + +-- now do first in n +SELECT * +INTO #temp_target_first +FROM #temp_target temp_cohort +WHERE (temp_cohort.time_between >= @limit_to_first_in_n_days OR temp_cohort.time_between = -1); + +-- now min prior obs +SELECT * +INTO #temp_target_prior +FROM #temp_target_first +WHERE datediff(day, observation_period_start_date, cohort_start_date) >= @min_prior_observation; + +-- now nesting +{@nesting_cohort_id != 0}?{SELECT +t.cohort_definition_id, +t.row_number, +t.subject_id, +t.cohort_start_date, +-- use the nesting end date if it is before the target end date +CASE WHEN t.cohort_end_date <= n.cohort_end_date THEN t.cohort_end_date +ELSE n.cohort_end_date END cohort_end_date, +t.observation_period_start_date, +t.observation_period_end_date, +t.char_type +INTO #temp_target_nest +FROM #temp_target_prior t +INNER JOIN +(SELECT * from @nesting_schema.@nesting_table +WHERE cohort_definition_id = @nesting_cohort_id) n +ON n.subject_id = t.subject_id +-- cohort starts between nesting date +AND n.cohort_start_date <= t.cohort_start_date +AND n.cohort_end_date >= t.cohort_start_date; +}:{ +SELECT * +INTO #temp_target_nest +FROM #temp_target_prior t; +} + +-- now age at start +SELECT * +INTO #temp_target_age +FROM #temp_target_nest t +INNER JOIN @cdm_database_schema.person p +ON p.person_id = t.subject_id +WHERE YEAR(t.cohort_start_date) - p.year_of_birth >= @min_age +AND YEAR(t.cohort_start_date) - p.year_of_birth <= @max_age; + +-- now gender +{@gender_concept_ids != ''}?{ +SELECT * +INTO #temp_target_gender +FROM #temp_target_age +INNER JOIN +@cdm_database_schema.person p +ON p.person_id = t.subject_id +WHERE p.gender_concept_id = '@gender_concept_ids'; +}:{ +SELECT * +INTO #temp_target_gender +FROM #temp_target_age; +} + +-- finally date: +{@study_start != '' | @study_end != ''}?{ +SELECT * +INTO #temp_target_date +FROM #temp_target_gender +WHERE 1 = 1 +{@study_start != ''}?{AND cohort_start_date >= CAST('@study_start' AS DATE)} +{@study_end != ''}?{AND cohort_start_date <= CAST('@study_end' AS DATE)} +; +} : { +SELECT * +INTO #temp_target_date +FROM #temp_target_gender; +} +-- ========================= -WHERE (temp_cohort.time_between >= @limit_to_first_in_n_days OR temp_cohort.time_between = -1) -AND datediff(day, op.observation_period_start_date, temp_cohort.cohort_start_date) >= @min_prior_observation; + +-- ========================= +-- ADDING FINAL COHORT INTO TABLE -- remove existing rows with cohort ids DELETE FROM @characterization_schema.@characterization_table WHERE char_type = 'target' -AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_target) +AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_target_date) ; -- insert the new rows @@ -58,26 +159,215 @@ AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_tar temp.row_number, temp.subject_id, temp.cohort_start_date, - temp.cohort_end_date, + temp.cohort_end_date, -- TODO: update cohort_end_date to be study_end_date if study_end_date is before? temp.observation_period_start_date, temp.observation_period_end_date, 'target' as char_type - FROM #temp_target temp; + FROM #temp_target_date temp; +-- ========================= -INSERT INTO @characterization_schema.@attrition_table -SELECT -cohort_definition_id, -'Target first in @limit_to_first_in_n_days - @min_prior_observation prior obs' as attr_reason, -count(*) as n +-- ========================= +-- DO ATTRITION - how to get 0 when there are no rows? +DELETE FROM @characterization_schema.@target_attrition_table +WHERE characterization_target_id in (SELECT DISTINCT cohort_definition_id FROM #temp_target_date) +; +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +1 as attr_order, -- new +'Target Start' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target +GROUP BY +cohort_definition_id +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +2 as attr_order, -- new +'First in @limit_to_first_in_n_days days' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people +FROM #temp_target_first +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +2 as attr_order, -- new +'First in @limit_to_first_in_n_days days' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_first) + +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +3 as attr_order, -- new +'With @min_prior_observation prior obs' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people +FROM #temp_target_prior +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +3 as attr_order, -- new +'With @min_prior_observation prior obs' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target_first -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_prior) + +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +4 as attr_order, -- new +'Nested in @nesting_cohort_id' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people +FROM #temp_target_nest +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +4 as attr_order, -- new +'Nested in @nesting_cohort_id' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target_prior -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_nest) + +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +5 as attr_order, -- new +'Aged @min_age to @max_age' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people +FROM #temp_target_age +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +5 as attr_order, -- new +'Aged @min_age to @max_age' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target_nest -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_age) + +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +6 as attr_order, -- new +'Gender in @gender_concept_ids' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people +FROM #temp_target_gender +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +6 as attr_order, -- new +'Gender in @gender_concept_ids' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target_age -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_gender) + +; + +INSERT INTO @characterization_schema.@target_attrition_table +SELECT +cohort_definition_id as characterization_target_id, +7 as attr_order, -- new +'Starting between @study_start to @study_end' as attr_reason, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new +FROM #temp_target_date +GROUP BY +cohort_definition_id + +-- add 0s +UNION +SELECT DISTINCT +cohort_definition_id as characterization_target_id, +7 as attr_order, -- new +'Starting between @study_start to @study_end' as attr_reason, +0 as n_events, +0 as n_people +FROM #temp_target_gender -- preious +WHERE cohort_definition_id NOT IN +(SELECT distinct cohort_definition_id FROM #temp_target_date) + +; + +-- ========================= + + +-- add final target count to table +DELETE FROM @characterization_schema.@target_count_table +WHERE characterization_target_id in (SELECT DISTINCT cohort_definition_id FROM #temp_target_date) +; + +INSERT INTO @characterization_schema.@target_count_table +SELECT +cohort_definition_id as characterization_target_id, +count(*) as n_events, -- new +count(distinct subject_id) as n_people -- new + +FROM #temp_target_date GROUP BY cohort_definition_id ; +-- ========================= -- clean up IF OBJECT_ID('tempdb..#temp_target', 'U') IS NOT NULL DROP TABLE #temp_target; +IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_fist; +IF OBJECT_ID('tempdb..#temp_target_prior', 'U') IS NOT NULL DROP TABLE #temp_target_prior; +IF OBJECT_ID('tempdb..#temp_target_nest', 'U') IS NOT NULL DROP TABLE #temp_target_nest; +IF OBJECT_ID('tempdb..#temp_target_age', 'U') IS NOT NULL DROP TABLE #temp_target_age; +IF OBJECT_ID('tempdb..#temp_target_gender', 'U') IS NOT NULL DROP TABLE #temp_target_gender; +IF OBJECT_ID('tempdb..#temp_target_date', 'U') IS NOT NULL DROP TABLE #temp_target_date; +-- ========================= + + + + diff --git a/inst/sql/sql_server/TargetCounts.sql b/inst/sql/sql_server/TargetCounts.sql deleted file mode 100644 index 97e4aab..0000000 --- a/inst/sql/sql_server/TargetCounts.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT cohort_definition_id/10, count(*) AS N -FROM @characterization_schema.@characterization_table -WHERE char_type = 'target' -GROUP BY cohort_definition_id, char_type; diff --git a/inst/sql/sql_server/TimeToEvent.sql b/inst/sql/sql_server/TimeToEvent.sql index e30e961..426b738 100644 --- a/inst/sql/sql_server/TimeToEvent.sql +++ b/inst/sql/sql_server/TimeToEvent.sql @@ -2,9 +2,9 @@ drop table if exists #targets; select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date asc) as era_number into #targets -from @target_database_schema.@target_table +from @characterization_schema.@characterization_table where cohort_definition_id in -(select distinct target_cohort_definition_id from #cohort_settings) +(select distinct characterization_target_id from #cohort_settings) ; drop table if exists #outcomes; @@ -49,7 +49,7 @@ inner join ( ) o1 on t1.subject_id = o1.subject_id inner join #cohort_settings ito1 -on t1.cohort_definition_id = ito1.target_cohort_definition_id +on t1.cohort_definition_id = ito1.characterization_target_id and o1.cohort_definition_id = ito1.outcome_cohort_definition_id ; @@ -77,50 +77,6 @@ group by --select * from #two_fu_bounds; -drop table if exists #t_prior_obs; -select - t1.cohort_definition_id, - 'Before target start' as observation_time_type, - datediff(day, t1.first_date, op1.observation_period_start_date) as time_to_event, - count(t1.subject_id) as num_persons -into #t_prior_obs -from ( - select cohort_definition_id, subject_id, min(cohort_start_date) as first_date - from #targets - group by cohort_definition_id, subject_id - ) t1 -inner join @cdm_database_schema.observation_period op1 -on t1.subject_id = op1.person_id -and t1.first_date >= op1.observation_period_start_date -and t1.first_date <= op1.observation_period_end_date -group by - t1.cohort_definition_id, - datediff(day, t1.first_date, op1.observation_period_start_date) -; - - -drop table if exists #t_post_obs; -select - t1.cohort_definition_id, - 'After target start' as observation_time_type, - datediff(day, t1.first_date, op1.observation_period_end_date) as time_to_event, - count(t1.subject_id) as num_persons -into #t_post_obs -from ( - select cohort_definition_id, subject_id, min(cohort_start_date) as first_date - from #targets - group by cohort_definition_id, subject_id - ) t1 -inner join @cdm_database_schema.observation_period op1 -on t1.subject_id = op1.person_id -and t1.first_date >= op1.observation_period_start_date -and t1.first_date <= op1.observation_period_end_date -group by - t1.cohort_definition_id, - datediff(day, t1.first_date, op1.observation_period_end_date) -; - - /*time-to-event distribution*/ drop table if exists #two_tte; @@ -262,7 +218,7 @@ from ( --daily counting for +/- 100 days, jenna comment why 100 select -target_cohort_definition_id, +target_cohort_definition_id as characterization_target_id, -- renamed outcome_cohort_definition_id, outcome_type, target_outcome_type, @@ -276,7 +232,7 @@ union all --30-day counting for +/- 1080 days (~ 3 years) select -target_cohort_definition_id, +target_cohort_definition_id as characterization_target_id, -- renamed outcome_cohort_definition_id, outcome_type, target_outcome_type, @@ -297,7 +253,7 @@ union all --365-day counting for +/- all days select -target_cohort_definition_id, +target_cohort_definition_id as characterization_target_id, -- renamed outcome_cohort_definition_id, outcome_type, target_outcome_type, @@ -315,9 +271,6 @@ group by target_cohort_definition_id, outcome_cohort_definition_id, outcome_type ) temp ; --- select * from #two_tte_summary; - - diff --git a/man/createCaseSeriesSettings.Rd b/man/createCaseSeriesSettings.Rd index 50475b8..0ed84af 100644 --- a/man/createCaseSeriesSettings.Rd +++ b/man/createCaseSeriesSettings.Rd @@ -5,10 +5,8 @@ \title{Create aggregate covariate study settings} \usage{ createCaseSeriesSettings( - targetIds, + studyPopulationSettings, outcomeIds, - limitToFirstInNDays = 99999, - minPriorObservation = 0, outcomeWashoutDays = 0, riskWindowStart = 1, startAnchor = "cohort start", @@ -23,14 +21,10 @@ createCaseSeriesSettings( ) } \arguments{ -\item{targetIds}{A list of cohortIds for the target cohorts} +\item{studyPopulationSettings}{A List of object created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria} \item{outcomeIds}{A list of cohortIds for the outcome cohorts} -\item{limitToFirstInNDays}{whether to limit each target cohort to the first entry into the cohort per N days per subject} - -\item{minPriorObservation}{The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index} - \item{outcomeWashoutDays}{Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} \item{riskWindowStart}{The start of the risk window (in days) relative to the `startAnchor`.} @@ -58,10 +52,12 @@ Create aggregate covariate study settings \examples{ caseSeriesSetting <- createCaseSeriesSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + minPriorObservation = 365, + limitToFirstInNDays = 365 + ), outcomeIds = c(3), - limitToFirstInNDays = 365, - minPriorObservation = 365, outcomeWashoutDays = 90, riskWindowStart = 1, startAnchor = "cohort start", diff --git a/man/createDechallengeRechallengeSettings.Rd b/man/createDechallengeRechallengeSettings.Rd index 5bae700..bc5388a 100644 --- a/man/createDechallengeRechallengeSettings.Rd +++ b/man/createDechallengeRechallengeSettings.Rd @@ -5,14 +5,14 @@ \title{Create dechallenge rechallenge study settings} \usage{ createDechallengeRechallengeSettings( - targetIds, + studyPopulationSettings, outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 ) } \arguments{ -\item{targetIds}{A list of cohortIds for the target cohorts} +\item{studyPopulationSettings}{An object created using \code{createStudyPopulationSettings} of a list of \code{createStudyPopulationSettings} that specifies cohort inclusion criteria} \item{outcomeIds}{A list of cohortIds for the outcome cohorts} @@ -28,7 +28,11 @@ Create dechallenge rechallenge study settings } \examples{ drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/man/createRiskFactorSettings.Rd b/man/createRiskFactorSettings.Rd index 2c08abe..6a63a60 100644 --- a/man/createRiskFactorSettings.Rd +++ b/man/createRiskFactorSettings.Rd @@ -5,10 +5,8 @@ \title{Create risk factor study settings} \usage{ createRiskFactorSettings( - targetIds, + studyPopulationSettings, outcomeIds, - limitToFirstInNDays = 99999, - minPriorObservation = 0, outcomeWashoutDays = 0, riskWindowStart = 1, startAnchor = "cohort start", @@ -34,14 +32,10 @@ createRiskFactorSettings( ) } \arguments{ -\item{targetIds}{A list of cohortIds for the target cohorts} +\item{studyPopulationSettings}{A list of objects created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria} \item{outcomeIds}{A list of cohortIds for the outcome cohorts} -\item{limitToFirstInNDays}{whether to limit each target cohort to the first entry into the cohort per N days per subject} - -\item{minPriorObservation}{The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index} - \item{outcomeWashoutDays}{Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} \item{riskWindowStart}{The start of the risk window (in days) relative to the `startAnchor`.} @@ -69,9 +63,12 @@ Create risk factor study settings \examples{ riskFactorSetting <- createRiskFactorSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + minPriorObservation = 365, + limitToFirstInNDays = 99999 + ), outcomeIds = c(3), - minPriorObservation = 365, outcomeWashoutDays = 90, riskWindowStart = 1, startAnchor = "cohort start", diff --git a/man/createStudyPopulationSettings.Rd b/man/createStudyPopulationSettings.Rd new file mode 100644 index 0000000..f225c3f --- /dev/null +++ b/man/createStudyPopulationSettings.Rd @@ -0,0 +1,56 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/StudyPopulation.R +\name{createStudyPopulationSettings} +\alias{createStudyPopulationSettings} +\title{create the study population settings} +\usage{ +createStudyPopulationSettings( + targetIds, + limitToFirstInNDays = 0, + minPriorObservation = 0, + nestingCohortId = NULL, + minAge = NULL, + maxAge = NULL, + studyStartDate = NULL, + studyEndDate = NULL, + genderConceptIds = NULL +) +} +\arguments{ +\item{targetIds}{A target cohort id or vector of target cohort ids to do the subsetting to} + +\item{limitToFirstInNDays}{Should only the first exposure in N days per subject be included?} + +\item{minPriorObservation}{The minimum required continuous observation time prior to index +date for a person to be included in the cohort.} + +\item{nestingCohortId}{A cohort definition id to restrict the target cohort. Patient in the target cohort +are only included if they are also in the nesting cohort at index.} + +\item{minAge}{The minimum age required to be in the target at index} + +\item{maxAge}{The maximum age required to be in the target at index} + +\item{studyStartDate}{The earliest date to be included into the target. Date format is 'yyyymmdd'.} + +\item{studyEndDate}{The latest date to be included into the target. Date format is 'yyyymmdd'.} + +\item{genderConceptIds}{A target cohort subject's gender concept to restrict to} +} +\value{ +A data.frame containing all the settings required +for creating the study populations of interest +} +\description{ +create the study population settings +} +\examples{ +# Create study population settings with a washout period of 365 days and + restricted to adults for target dates that occur for the first time in 365 days. +populationSettings <- createStudyPopulationSettings( + targetId = 1, + limitToFirstInNDays = 365, + minPriorObservation = 365, + minAge = 18 + ) +} diff --git a/man/createTargetBaselineSettings.Rd b/man/createTargetBaselineSettings.Rd index fbd5406..18cfbc4 100644 --- a/man/createTargetBaselineSettings.Rd +++ b/man/createTargetBaselineSettings.Rd @@ -5,9 +5,7 @@ \title{Create target baseline aggregate covariate study settings} \usage{ createTargetBaselineSettings( - targetIds, - limitToFirstInNDays = 99999, - minPriorObservation = 0, + studyPopulationSettings, covariateSettings = FeatureExtraction::createCovariateSettings(useDemographicsGender = TRUE, useDemographicsAge = TRUE, useDemographicsAgeGroup = TRUE, useDemographicsRace = TRUE, useDemographicsEthnicity = TRUE, useDemographicsIndexYear = TRUE, @@ -26,11 +24,7 @@ createTargetBaselineSettings( ) } \arguments{ -\item{targetIds}{A list of cohortIds for the target cohorts} - -\item{limitToFirstInNDays}{Whether to remove target cohort entries that occur within limitToFirstInNDays of a prior entry. limitToFirstInNDays = 99999 means limit to first entry.} - -\item{minPriorObservation}{The minimum time (in days) in the database a patient in the target cohorts must be observed prior to index} +\item{studyPopulationSettings}{An object created using \code{createStudyPopulationSettings} or a list of \code{createStudyPopulationSettings} that specifies specific populations of interest} \item{covariateSettings}{An object created using \code{FeatureExtraction::createCovariateSettings}} } @@ -43,9 +37,11 @@ Create target baseline aggregate covariate study settings \examples{ aggregateSetting <- createTargetBaselineSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 1:2, limitToFirstInNDays = 99999, minPriorObservation = 365 + ) ) } diff --git a/man/createTimeToEventSettings.Rd b/man/createTimeToEventSettings.Rd index 275b65c..f8c60ff 100644 --- a/man/createTimeToEventSettings.Rd +++ b/man/createTimeToEventSettings.Rd @@ -4,10 +4,10 @@ \alias{createTimeToEventSettings} \title{Create time to event study settings} \usage{ -createTimeToEventSettings(targetIds, outcomeIds) +createTimeToEventSettings(studyPopulationSettings, outcomeIds) } \arguments{ -\item{targetIds}{A list of cohortIds for the target cohorts} +\item{studyPopulationSettings}{An object created using \code{createStudyPopulationSettings} or a list of \code{createStudyPopulationSettings} that specifies cohort inclusion criteria} \item{outcomeIds}{A list of cohortIds for the outcome cohorts} } @@ -21,7 +21,11 @@ Create time to event study settings # example code tteSet <- createTimeToEventSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index 7a13108..ce8b068 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -10,6 +10,8 @@ runCharacterizationAnalyses( targetTable, outcomeDatabaseSchema, outcomeTable, + nestingCohortTable, + nestingCohortDatabaseSchema, outputDatabaseSchema = targetDatabaseSchema, outputTable = "characterization_cohort", tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), @@ -25,7 +27,7 @@ runCharacterizationAnalyses( threads = 1, cohortGenerationThreads = NULL, nTargetJobs = 1, - minCharacterizationMean = 0.01, + minCharacterizationMean = 0.001, minCovariateCount = 0, mode = "CohortIncidence", minSMD = 0 @@ -46,6 +48,10 @@ this should include both the database and schema name, for example \item{outcomeTable}{Name of the outcome cohort table.} +\item{nestingCohortTable}{The cohort table to extract the nesting cohort from} + +\item{nestingCohortDatabaseSchema}{The schema containing the nestingCohortTable} + \item{outputDatabaseSchema}{The schema where the characterization cohort table will be saved into} \item{outputTable}{The table name where the characterization cohort table will be saved into} diff --git a/man/viewCharacterization.Rd b/man/viewCharacterization.Rd index 10d7b78..6a1f710 100644 --- a/man/viewCharacterization.Rd +++ b/man/viewCharacterization.Rd @@ -25,7 +25,9 @@ Input is the output of ... conDet <- exampleOmopConnectionDetails() tteSet <- createTimeToEventSettings( - targetIds = c(1,2), +studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2) + ), outcomeIds = 3 ) diff --git a/tests/testthat/test-CohortGeneration.R b/tests/testthat/test-CohortGeneration.R index 5048087..53f4e19 100644 --- a/tests/testthat/test-CohortGeneration.R +++ b/tests/testthat/test-CohortGeneration.R @@ -6,30 +6,40 @@ test_that("getCohortJobs", { outcomeIds <- c(3) timeToEventSettings1 <- createTimeToEventSettings( - targetIds = 1, + createStudyPopulationSettings( + targetIds = 1 + ), outcomeIds = c(3, 4) ) timeToEventSettings2 <- createTimeToEventSettings( - targetIds = 2, + createStudyPopulationSettings( + targetIds = 2 + ), outcomeIds = c(3, 4) ) dechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 ) targetBaselineSettings1 <- createTargetBaselineSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds + ), covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE ) ) targetBaselineSettings2 <- createTargetBaselineSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds + ), covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsAge = TRUE, useDemographicsRace = TRUE @@ -37,7 +47,11 @@ test_that("getCohortJobs", { ) riskFactorSettings <- createRiskFactorSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 365 + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -51,7 +65,11 @@ test_that("getCohortJobs", { ) caseSeriesSettings <- createCaseSeriesSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 365 + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -86,9 +104,9 @@ jobs <- getCohortJobs( nTargetJobs = 1 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 2) +testthat::expect_true(nrow(jobs$jobs) == 3) testthat::expect_true(sum(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds %in% targetIds) == 3) @@ -99,9 +117,9 @@ jobs <- getCohortJobs( nTargetJobs = 2 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 4) +testthat::expect_true(nrow(jobs$jobs) == 6) testthat::expect_true(sum(unique(c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, ParallelLogger::convertJsonToSettings(jobs$jobs$settings[2])$targetIds)) %in% targetIds) == 3) @@ -112,9 +130,9 @@ jobs <- getCohortJobs( nTargetJobs = 3 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 6) +testthat::expect_true(nrow(jobs$jobs) == 9) testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, @@ -129,9 +147,9 @@ jobs <- getCohortJobs( nTargetJobs = 4 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 6) +testthat::expect_true(nrow(jobs$jobs) == 9) testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, @@ -147,9 +165,9 @@ jobs <- getCohortJobs( nTargetJobs = 4 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 9) +testthat::expect_true(nrow(jobs$jobs) == 12) testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, @@ -158,15 +176,15 @@ testthat::expect_true(sum(unique( ) == 3) -jobs <- getCohortJobs( +jobs <- Characterization:::getCohortJobs( characterizationSettings = characterizationSettings, mode = 'PatientLevelPrediction', nTargetJobs = 4 ) -testthat::expect_true(nrow(jobs$targets) == 3) +testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 9) +testthat::expect_true(nrow(jobs$jobs) == 12) testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, diff --git a/tests/testthat/test-ExportingCsvFiles.R b/tests/testthat/test-ExportingCsvFiles.R index 3e455d8..165250b 100644 --- a/tests/testthat/test-ExportingCsvFiles.R +++ b/tests/testthat/test-ExportingCsvFiles.R @@ -774,7 +774,8 @@ test_that("exportAttrition", { start_anchor = rep('cohort_start',3), risk_window_end = rep(365,3), end_anchor = rep('cohort_start',3), - runtype = rep('PLP',3), + risk_factor_settings = rep(TRUE,3), + case_series_settings = rep(FALSE,3), characterization_case_id = c(1,2,3), setting_id = 'set1', characterization_target_id = c(10,20,40), diff --git a/tests/testthat/test-StudyPopulation.R b/tests/testthat/test-StudyPopulation.R new file mode 100644 index 0000000..2fbb69f --- /dev/null +++ b/tests/testthat/test-StudyPopulation.R @@ -0,0 +1,53 @@ + +test_that("replaceNull", { + + testthat::expect_equal(replaceNull(0,545),0) + testthat::expect_equal(replaceNull(NULL,545),545) + + testthat::expect_equal(replaceNull('0','545'),'0') + testthat::expect_equal(replaceNull(NULL,'545'),'545') + +}) + +test_that("createStudyPopulationSettings", { + + set <- createStudyPopulationSettings(targetIds = c(232,23)) + testthat::expect_true(nrow(set) == 2) + + set <- createStudyPopulationSettings(targetIds = c(232,23,232)) + testthat::expect_true(nrow(set) == 2) + + set <- createStudyPopulationSettings(targetIds = 232, + nestingCohortId = 12) + testthat::expect_true(nrow(set) == 1) + testthat::expect_true(set$targetId == 232) + testthat::expect_true(set$nestingCohortId == 12) + + set <- createStudyPopulationSettings(targetIds = 232, minAge = 18) + testthat::expect_true(nrow(set) == 1) + testthat::expect_true(set$targetId == 232) + testthat::expect_true(set$minAge == 18) + + set <- createStudyPopulationSettings(targetIds = 23, minAge = 18, + genderConceptIds = c(32434,1212)) + testthat::expect_true(nrow(set) == 1) + testthat::expect_true(set$targetId == 23) + testthat::expect_true(set$minAge == 18) + testthat::expect_true(set$genderConceptIds == '1212,32434') + + +}) + +test_that("combineStudyPopulationSettings", { + set <- list( + createStudyPopulationSettings(targetIds = c(232,23,232)), + createStudyPopulationSettings(targetIds = 232, + nestingCohortId = 12), + createStudyPopulationSettings(targetIds = 232, minAge = 18), + createStudyPopulationSettings(targetIds = 23, minAge = 18, genderConceptIds = c(32434,1212)) + ) + + res <- combineStudyPopulationSettings(set) + testthat::expect_true(nrow(res) == 5) + testthat::expect_true('targetId' %in% colnames(res)) +}) diff --git a/tests/testthat/test-dechallengeRechallenge.R b/tests/testthat/test-dechallengeRechallenge.R index 093bca6..3a70bac 100644 --- a/tests/testthat/test-dechallengeRechallenge.R +++ b/tests/testthat/test-dechallengeRechallenge.R @@ -15,7 +15,11 @@ test_that("createDechallengeRechallengeSettings", { outcomeIds <- sample(x = 100, size = sample(10, 1)) res <- createDechallengeRechallengeSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 @@ -26,7 +30,7 @@ test_that("createDechallengeRechallengeSettings", { ) testthat::expect_equal( - res$targetCohortDefinitionIds, + res$studyPopulationSettings$targetId, targetIds ) @@ -50,19 +54,34 @@ test_that("computeDechallengeRechallengeAnalyses", { targetIds <- c(2) outcomeIds <- c(3, 4) - res <- createDechallengeRechallengeSettings( - targetIds = targetIds, + drSet <- createDechallengeRechallengeSettings( + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 ) + charSet <- createCharacterizationSettings( + dechallengeRechallengeSettings = drSet + ) + + charSet$dechallengeRechallengeSettings[[1]]$characterizationTargetIds <- 2 + + # make the cohorts in a table dcLoc <- tempfile("runADechal") - dc <- computeDechallengeRechallengeAnalyses( + dc <- Characterization::computeDechallengeRechallengeAnalyses( connectionDetails = connectionDetails, - targetDatabaseSchema = "main", - targetTable = "cohort", - settings = res, + #targetDatabaseSchema = "main", + #targetTable = "cohort", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort", + characterizationDatabaseSchema = "main", + characterizationTable = "cohort", + settings = charSet$dechallengeRechallengeSettings[[1]], databaseId = "testing", outputFolder = dcLoc ) @@ -124,19 +143,30 @@ test_that("computeDechallengeRechallengeAnalyses", { DatabaseConnector::disconnect(con) - res <- createDechallengeRechallengeSettings( + drSet <- createDechallengeRechallengeSettings( targetIds = 1, outcomeIds = 2, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 ) + charSet <- createCharacterizationSettings( + dechallengeRechallengeSettings = drSet + ) + + charSet$dechallengeRechallengeSettings[[1]]$characterizationTargetIds <- 1 + + dcLoc <- tempfile("runADechal2") dc <- computeDechallengeRechallengeAnalyses( connectionDetails = connectionDetailsReal, - targetDatabaseSchema = "main", - targetTable = "cohort_dechal", - settings = res, + #targetDatabaseSchema = "main", + #targetTable = "cohort_dechal", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort_dechal", + characterizationDatabaseSchema = "main", + characterizationTable = "cohort_dechal", + settings = charSet$dechallengeRechallengeSettings[[1]], databaseId = "testing", outputFolder = dcLoc ) @@ -208,21 +238,74 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { ) DatabaseConnector::disconnect(con) - set <- createDechallengeRechallengeSettings( - targetIds = 1, + drSet <- createDechallengeRechallengeSettings( + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 1 + ), outcomeIds = 2, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 # 31 ) + charSet <- createCharacterizationSettings( + dechallengeRechallengeSettings = drSet + ) + + # add the target_settings table + con <- DatabaseConnector::connect(connectionDetails = connectionDetailsReal) + + DatabaseConnector::insertTable( + data = data.frame( + characterizationTargetIds = c(10), + targetId = c(1) + ), + connection = con, + databaseSchema = "main", + tableName = "target_settings", + createTable = TRUE, + dropTableIfExists = TRUE, + camelCaseToSnakeCase = FALSE + ) + DatabaseConnector::disconnect(con) + + # add the characterization cohort table "characterization" + con <- DatabaseConnector::connect(connectionDetails = connectionDetailsReal) + + DatabaseConnector::insertTable( + data = data.frame( + cohort_definition_id = rep(10, 10), + subject_id = c(1, 2, 2, 2, 2, 3, 3, 3, 4, 4), + cohort_start_date = as.Date(c( + "2001-01-01", + "2001-01-01", "2001-03-14", "2001-05-01", "2001-07-01", + "2001-01-01", "2001-03-01", "2001-05-01", + "2001-01-01", "2001-03-01" + )), + cohort_end_date = as.Date(c( + "2001-01-31", + "2001-01-31", "2001-03-16", "2001-05-30", "2001-07-31", + "2001-01-31", "2001-03-30", "2001-05-30", + "2001-01-31", "2001-03-30" + )) + ), + connection = con, + databaseSchema = "main", + tableName = "characterization", + createTable = TRUE, + dropTableIfExists = TRUE, + camelCaseToSnakeCase = FALSE + ) + DatabaseConnector::disconnect(con) dcLoc <- tempfile("runADechal2") dc <- computeRechallengeFailCaseSeriesAnalyses( connectionDetails = connectionDetailsReal, targetDatabaseSchema = "main", targetTable = "cohort", - settings = set, + settings = charSet$dechallengeRechallengeSettings[[1]], outcomeDatabaseSchema = "main", outcomeTable = "cohort", + characterizationDatabaseSchema = "main", + characterizationTable = "characterization", databaseId = "testing", outputFolder = dcLoc ) @@ -236,11 +319,14 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { testthat::expect_true(is.na(dc$subjectId)) dcLoc <- tempfile("runADechal3") - dc <- computeRechallengeFailCaseSeriesAnalyses( + dc <- Characterization::computeRechallengeFailCaseSeriesAnalyses( connectionDetails = connectionDetailsReal, targetDatabaseSchema = "main", targetTable = "cohort", - settings = set, + characterizationDatabaseSchema = "main", + characterizationTable = "characterization", + targetSettingsTable = "target_settings", # new + settings = charSet$dechallengeRechallengeSettings[[1]], outcomeDatabaseSchema = "main", outcomeTable = "cohort", databaseId = "testing", @@ -262,20 +348,24 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { # add test for job creation code -test_that("computeDechallengeRechallengeAnalyses", { +test_that("getDechallengeRechallengeJobs", { targetIds <- c(2, 5, 6, 7, 8) outcomeIds <- c(3, 4, 9, 10) res <- createDechallengeRechallengeSettings( - targetIds = targetIds, + createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 ) + charSettings <- createCharacterizationSettings( + dechallengeRechallengeSettings = res + ) + jobs <- getDechallengeRechallengeJobs( - characterizationSettings = createCharacterizationSettings( - dechallengeRechallengeSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 1 ) @@ -286,17 +376,22 @@ test_that("computeDechallengeRechallengeAnalyses", { targetIdFromSettings <- do.call( what = unique, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$targetCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$characterizationTargetIds }) ) - testthat::expect_true(sum(targetIds %in% targetIdFromSettings) == + + originalTs <- charSettings$characterizationTargetLookup$targetId[ + charSettings$characterizationTargetLookup$characterizationTargetId %in% targetIdFromSettings + ] + + testthat::expect_true(sum(targetIds %in% originalTs) == length(targetIds)) # check all outcome ids are in there outcomeIdFromSettings <- do.call( what = unique, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeIds }) ) testthat::expect_true(sum(outcomeIds %in% outcomeIdFromSettings) == @@ -305,9 +400,7 @@ test_that("computeDechallengeRechallengeAnalyses", { # checking more threads 3 jobs <- getDechallengeRechallengeJobs( - characterizationSettings = createCharacterizationSettings( - dechallengeRechallengeSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 3 ) @@ -318,17 +411,22 @@ test_that("computeDechallengeRechallengeAnalyses", { targetIdFromSettings <- do.call( what = c, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$targetCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$characterizationTargetIds }) ) - testthat::expect_true(sum(targetIds %in% targetIdFromSettings) == + + originalTs <- charSettings$characterizationTargetLookup$targetId[ + charSettings$characterizationTargetLookup$characterizationTargetId %in% targetIdFromSettings + ] + + testthat::expect_true(sum(targetIds %in% originalTs) == length(targetIds)) # check all outcome ids are in there outcomeIdFromSettings <- do.call( what = c, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeIds }) ) testthat::expect_true(sum(outcomeIds %in% outcomeIdFromSettings) == @@ -338,9 +436,7 @@ test_that("computeDechallengeRechallengeAnalyses", { # checking more threads than needed 20 jobs <- getDechallengeRechallengeJobs( - characterizationSettings = createCharacterizationSettings( - dechallengeRechallengeSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 20 ) @@ -351,17 +447,20 @@ test_that("computeDechallengeRechallengeAnalyses", { targetIdFromSettings <- do.call( what = c, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$targetCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$characterizationTargetIds }) ) - testthat::expect_true(sum(targetIds %in% targetIdFromSettings) == + originalTs <- charSettings$characterizationTargetLookup$targetId[ + charSettings$characterizationTargetLookup$characterizationTargetId %in% targetIdFromSettings + ] + testthat::expect_true(sum(targetIds %in% originalTs) == length(targetIds)) # check all outcome ids are in there outcomeIdFromSettings <- do.call( what = c, args = lapply(1:nrow(jobs), function(i) { - ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeCohortDefinitionIds + ParallelLogger::convertJsonToSettings(jobs$settings[i])$outcomeIds }) ) testthat::expect_true(sum(outcomeIds %in% outcomeIdFromSettings) == diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index b1792e1..7db6b0f 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -8,30 +8,40 @@ test_that("runCharacterizationAnalyses", { outcomeIds <- c(3) timeToEventSettings1 <- createTimeToEventSettings( - targetIds = 1, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 1 + ), outcomeIds = c(3, 4) ) timeToEventSettings2 <- createTimeToEventSettings( - targetIds = 2, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 2 + ), outcomeIds = c(3, 4) ) dechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 ) targetBaselineSettings1 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE ) ) targetBaselineSettings2 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsAge = TRUE, useDemographicsRace = TRUE @@ -39,7 +49,9 @@ test_that("runCharacterizationAnalyses", { ) riskFactorSettings <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -53,7 +65,9 @@ test_that("runCharacterizationAnalyses", { ) caseSeriesSettings <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -88,9 +102,23 @@ test_that("runCharacterizationAnalyses", { testthat::expect_true( length(characterizationSettings$timeToEventSettings) == 2 ) + testthat::expect_true( + length(characterizationSettings$timeToEventSettings[[1]]$characterizationTargetIds) == 1 + ) + testthat::expect_true( + is.null(characterizationSettings$timeToEventSettings[[1]]$studyPopulationSettings) + ) + testthat::expect_true( length(characterizationSettings$dechallengeRechallengeSettings) == 1 ) + testthat::expect_true( + length(characterizationSettings$dechallengeRechallengeSettings[[1]]$characterizationTargetIds) == 3 + ) + testthat::expect_true( + is.null(characterizationSettings$dechallengeRechallengeSettings[[1]]$studyPopulationSettings) + ) + testthat::expect_true( length(characterizationSettings$targetBaselineSettings) == 2 ) @@ -133,7 +161,6 @@ test_that("runCharacterizationAnalyses", { tempFolder <- tempfile("Characterization") on.exit(unlink(tempFolder, recursive = TRUE), add = TRUE) - runCharacterizationAnalyses( connectionDetails = connectionDetails, cdmDatabaseSchema = "main", diff --git a/tests/testthat/test-targetAnalysis.R b/tests/testthat/test-targetAnalysis.R index fd0cfe8..925c9b5 100644 --- a/tests/testthat/test-targetAnalysis.R +++ b/tests/testthat/test-targetAnalysis.R @@ -16,15 +16,17 @@ test_that("createTargetBaselineSettings", { ) res <- createTargetBaselineSettings( - targetIds = targetIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10, + ), covariateSettings = covariateSettings ) testthat::expect_equal( - res$targetIds, - targetIds + unique(res$studyPopulationSettings$targetId), + unique(targetIds) ) testthat::expect_equal( res$covariateSettings[[1]], @@ -32,12 +34,12 @@ test_that("createTargetBaselineSettings", { ) testthat::expect_equal( - res$minPriorObservation, + unique(res$studyPopulationSettings$minPriorObservation), 10 ) testthat::expect_equal( - res$limitToFirstInNDays, + unique(res$studyPopulationSettings$limitToFirstInNDays), 9 ) @@ -49,9 +51,11 @@ test_that("error when using temporal features", { testthat::expect_error( createTargetBaselineSettings( - targetIds = targetIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10, + ), covariateSettings = temporalCovariateSettings ) ) @@ -63,9 +67,11 @@ test_that("error when using temporal features", { testthat::expect_error( createTargetBaselineSettings( - targetIds = targetIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10, + ), covariateSettings = temporalCovariateSettings ) ) @@ -84,13 +90,15 @@ test_that("createTargetBaselineSettings covariateList", { covariateSettings <- list(covariateSettings1, covariateSettings2) res <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), covariateSettings = covariateSettings ) testthat::expect_equal( - res$targetIds, - targetIds + unique(res$studyPopulationSettings$targetId), + unique(targetIds) ) testthat::expect_equal( res$covariateSettings, @@ -110,10 +118,14 @@ test_that("getTargetBaselineJobs", { minPriorObservation <- sample(30, 1) limitToFirstInNDays <- sample(300, 1) - res <- createTargetBaselineSettings( + studyPop <- createStudyPopulationSettings( targetIds = targetIds, minPriorObservation = minPriorObservation, - limitToFirstInNDays = limitToFirstInNDays, + limitToFirstInNDays = limitToFirstInNDays + ) + + res <- createTargetBaselineSettings( + studyPopulationSettings = studyPop, covariateSettings = covariateSettings ) @@ -130,7 +142,7 @@ test_that("getTargetBaselineJobs", { testthat::expect_true(nrow(jobDf) == 1) testthat::expect_true( - paste(c("t_1", minPriorObservation,limitToFirstInNDays), collapse ='_') %in% jobDf$executionFolder + paste("t_1", collapse ='_') %in% jobDf$executionFolder ) settings <- ParallelLogger::convertJsonToSettings(jobDf$settings[1]) @@ -157,25 +169,35 @@ test_that("getTargetBaselineJobs", { testthat::expect_true( sum(c( - paste0(c("t_1", minPriorObservation,limitToFirstInNDays), collapse = '_'), - paste0(c("t_2", minPriorObservation,limitToFirstInNDays), collapse = '_') + paste0("t_1", collapse = '_'), + paste0("t_2", collapse = '_') ) %in% jobDf$executionFolder) == 2 ) # now check nTargetJobs = 3 + charSettings <- createCharacterizationSettings( + targetBaselineSettings = res + ) jobDf <- getTargetBaselineJobs( - characterizationSettings = createCharacterizationSettings( - targetBaselineSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 3 ) testthat::expect_true(nrow(jobDf) == 3) # check the target ids - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$targetIds == targetIds[1]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$targetIds == targetIds[2]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$targetIds == targetIds[3]) + tId1 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[1] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$characterizationTargetIds == tId1) + tId2 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[2] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$characterizationTargetIds == tId2) + tId3 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[3] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$characterizationTargetIds == tId3) # now check nTargetJobs = 4 jobDf <- getTargetBaselineJobs( @@ -206,9 +228,11 @@ test_that("computeTargetBaselineAnalyses", { ) res <- createTargetBaselineSettings( - targetIds = targetIds, - limitToFirstInNDays = 365, - minPriorObservation = 30, + createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 30 + ), covariateSettings = covariateSettings ) From d2c78c94b818e72fc11a598d134fa5ebebcefe02 Mon Sep 17 00:00:00 2001 From: jreps Date: Mon, 1 Jun 2026 22:19:33 -0400 Subject: [PATCH 03/39] minor fixes minor fixes --- R/CohortGeneration.R | 2 +- R/DechallengeRechallenge.R | 8 ++++---- R/LookupCohortSettings.R | 15 +++++++++++++-- inst/sql/sql_server/RechallengeFailCaseSeries.sql | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index a3fbb8a..a006774 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -647,7 +647,7 @@ generateCases <- function( characterization_schema = characterizationDatabaseSchema, characterization_table = characterizationTable, - case_attrition_table = caseAttritionTable, + #case_attrition_table = caseAttritionTable, case_count_table = caseCountTable, case_settings_schema = characterizationDatabaseSchema, case_settings_table = caseSettingsTable, diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index 99a1872..c0f504d 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -345,10 +345,10 @@ computeRechallengeFailCaseSeriesAnalyses <- function( tempEmulationSchema = tempEmulationSchema, errorMessages = errorMessages ) - .checkDechallengeRechallengeSettings( - settings = settings, - errorMessages = errorMessages - ) + #.checkDechallengeRechallengeSettings( + # settings = settings, + # errorMessages = errorMessages + #) valid <- checkmate::reportAssertions(errorMessages) diff --git a/R/LookupCohortSettings.R b/R/LookupCohortSettings.R index b7640c0..12f50f8 100644 --- a/R/LookupCohortSettings.R +++ b/R/LookupCohortSettings.R @@ -65,9 +65,21 @@ lookupCases <- function( risk_window_end FROM @lookup_schema.@lookup_table lt - INNER JOIN @lookup_schema.@case_count_table cct + + INNER JOIN + (SELECT * FROM @lookup_schema.@case_count_table + WHERE cohort_type = 'Cases' + AND n_people >= @min_case_size + ) cct ON lt.characterization_case_id = cct.characterization_case_id + INNER JOIN + (SELECT * FROM @lookup_schema.@case_count_table + WHERE cohort_type = 'non-cases' + AND n_people >= @min_case_size + ) ncct + ON lt.characterization_case_id = ncct.characterization_case_id + WHERE lt.characterization_target_id in (@char_ids) AND lt.outcome_id in (@outcome_ids) AND lt.outcome_washout_days = @outcome_washout_days @@ -75,7 +87,6 @@ lookupCases <- function( AND lt.risk_window_start = @risk_window_start AND lt.end_anchor = '@end_anchor' AND lt.risk_window_end = @risk_window_end - AND cct.n_people >= @min_case_size ; " diff --git a/inst/sql/sql_server/RechallengeFailCaseSeries.sql b/inst/sql/sql_server/RechallengeFailCaseSeries.sql index a80a699..4b1c250 100644 --- a/inst/sql/sql_server/RechallengeFailCaseSeries.sql +++ b/inst/sql/sql_server/RechallengeFailCaseSeries.sql @@ -13,9 +13,9 @@ tc.cohort_end_date, CASE WHEN sc.subject_id is NULL THEN 0 ELSE 1 END included INTO #target_cohort FROM @target_database_schema.@target_table tc -INNER JOIN @characterization_schema.@target_settings ts +INNER JOIN @characterization_database_schema.@target_settings ts ON tc.cohort_definition_id = ts.target_id -LEFT JOIN @characterization_schema.@characterization_table sc +LEFT JOIN @characterization_database_schema.@characterization_table sc ON sc.subject_id = tc.subject_id AND sc.cohort_start_date = tc.cohort_start_date AND sc.cohort_definition_id = ts.characterization_target_id From 9d20b38045782d4a528ac85b6db949968d51889d Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 4 Jun 2026 16:56:59 -0400 Subject: [PATCH 04/39] initial unit test updates - updating vignettes - updating unit tests for main analyses and fixing issues (still need to update results database via migration scripts) --- NAMESPACE | 3 - R/CaseSeries.R | 3 +- R/CohortGeneration.R | 8 +- R/DechallengeRechallenge.R | 81 +------- R/ExportingCsvFiles.R | 73 +++++++ R/LookupCohortSettings.R | 10 +- R/RiskFactorAnalysis.R | 5 +- R/RunCharacterization.R | 93 ++++++--- R/StudyPopulation.R | 2 +- R/TimeToEvent.R | 41 +--- README.md | 6 +- .../resultsDataModelSpecification.csv | 2 + .../sql_server/RechallengeFailCaseSeries.sql | 2 +- inst/sql/sql_server/TargetCohorts.sql | 2 +- man/computeDechallengeRechallengeAnalyses.Rd | 84 -------- ...omputeRechallengeFailCaseSeriesAnalyses.Rd | 89 --------- man/computeTimeToEventAnalyses.Rd | 92 --------- man/createCharacterizationSettings.Rd | 6 +- man/createDechallengeRechallengeSettings.Rd | 5 - man/createRiskFactorSettings.Rd | 8 +- man/createStudyPopulationSettings.Rd | 2 +- man/createTimeToEventSettings.Rd | 4 - man/loadCharacterizationSettings.Rd | 6 +- man/runCharacterizationAnalyses.Rd | 12 +- man/saveCharacterizationSettings.Rd | 6 +- tests/testthat/setup.R | 26 +++ tests/testthat/test-CaseSeries.R | 92 ++++++--- tests/testthat/test-ExportingCsvFiles.R | 104 ++-------- tests/testthat/test-RiskFactor.R | 82 +++++--- tests/testthat/test-dbs.R | 20 +- tests/testthat/test-dechallengeRechallenge.R | 189 +++++++++++++----- tests/testthat/test-runCharacterization.R | 65 +++--- tests/testthat/test-targetAnalysis.R | 4 + tests/testthat/test-timeToEvent.R | 58 +++++- vignettes/Specification.Rmd | 34 ++-- vignettes/UsingPackage.Rmd | 48 +++-- 36 files changed, 636 insertions(+), 731 deletions(-) delete mode 100644 man/computeDechallengeRechallengeAnalyses.Rd delete mode 100644 man/computeRechallengeFailCaseSeriesAnalyses.Rd delete mode 100644 man/computeTimeToEventAnalyses.Rd diff --git a/NAMESPACE b/NAMESPACE index acf4ed9..afa5c5e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,9 +2,6 @@ export(cleanIncremental) export(cleanNonIncremental) -export(computeDechallengeRechallengeAnalyses) -export(computeRechallengeFailCaseSeriesAnalyses) -export(computeTimeToEventAnalyses) export(createCaseSeriesSettings) export(createCharacterizationSettings) export(createCharacterizationTables) diff --git a/R/CaseSeries.R b/R/CaseSeries.R index fef094e..56f9757 100644 --- a/R/CaseSeries.R +++ b/R/CaseSeries.R @@ -209,7 +209,8 @@ computeCaseSeriesAnalyses <- function( riskWindowStart = settings$riskWindowStart, endAnchor = settings$endAnchor, riskWindowEnd = settings$riskWindowEnd, - minCaseSize = minCaseSize + minCaseSize = minCaseSize, + applyMinSizeToNonCases = FALSE ) completionTime <- Sys.time() - start diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index a006774..2762b6f 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -10,8 +10,8 @@ generateCohorts <- function( targetTable, outcomeDatabaseSchema, outcomeTable, - nestingCohortDatabaseSchema, - nestingCohortTable, + nestingCohortDatabaseSchema = targetDatabaseSchema, + nestingCohortTable = targetTable, outputDatabaseSchema = targetDatabaseSchema, outputTable = 'characterization_cohort', cdmDatabaseSchema, @@ -71,8 +71,8 @@ generateCohorts <- function( connection = connection, databaseSchema = outputDatabaseSchema, tableName = targetSettingsTableWithHash, - dropTableIfExists = FALSE, - createTable = FALSE, + dropTableIfExists = TRUE, # changed from FALSE, + createTable = TRUE, # changed from FALSE, data = cohortJobs$targets, camelCaseToSnakeCase = TRUE, progressBar = progressBar diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index c0f504d..7be864a 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -93,43 +93,7 @@ createDechallengeRechallengeSettings <- function( return(result) } -#' Compute dechallenge rechallenge study -#' -#' @template ConnectionDetails -#' @template TargetOutcomeTables -#' @param characterizationDatabaseSchema ... -#' @param characterizationTable ... -#' @template TempEmulationSchema -#' @param settings The settings for the timeToEvent study -#' @param databaseId An identifier for the database (string) -#' @param outputFolder A directory to save the results as csv files -#' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 -#' @param progressBar Whether to display a progress bar while the analysis is running -#' @param ... extra inputs -#' @family DechallengeRechallenge -#' -#' @return -#' An \code{Andromeda::andromeda()} object containing the dechallenge rechallenge results -#' -#' @examples -#' -#' conDet <- exampleOmopConnectionDetails() -#' -#' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), -#' outcomeIds = 3 -#' ) -#' -#' computeDechallengeRechallengeAnalyses( -#' connectionDetails = conDet, -#' targetDatabaseSchema = 'main', -#' targetTable = 'cohort', -#' settings = drSet, -#' outputFolder = tempdir() -#' ) -#' -#' -#' @export + computeDechallengeRechallengeAnalyses <- function( connectionDetails = NULL, targetDatabaseSchema, # not needed @@ -203,7 +167,7 @@ computeDechallengeRechallengeAnalyses <- function( outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow ) @@ -265,45 +229,6 @@ computeDechallengeRechallengeAnalyses <- function( } -#' Compute fine the subjects that fail the dechallenge rechallenge study -#' -#' @template ConnectionDetails -#' @template TargetOutcomeTables -#' @param characterizationDatabaseSchema ... -#' @param characterizationTable ... -#' @param targetSettingsTable The name of the target settings table -#' @template TempEmulationSchema -#' @param settings The settings for the timeToEvent study -#' @param databaseId An identifier for the database (string) -#' @param showSubjectId if F then subject_ids are hidden (recommended if sharing results) -#' @param outputFolder A directory to save the results as csv files -#' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 -#' @param progressBar Whether to display a progress bar while the analysis is running -#' @param executionId a unique id for the run -#' @param ... extra inputs -#' @family DechallengeRechallenge -#' -#' @return -#' An \code{Andromeda::andromeda()} object with the case series details of the failed rechallenge -#' -#' @examples -#' -#' conDet <- exampleOmopConnectionDetails() -#' -#' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), -#' outcomeIds = 3 -#' ) -#' -#' computeRechallengeFailCaseSeriesAnalyses( -#' connectionDetails = conDet, -#' targetDatabaseSchema = 'main', -#' targetTable = 'cohort', -#' settings = drSet, -#' outputFolder = tempdir() -#' ) -#' -#' @export computeRechallengeFailCaseSeriesAnalyses <- function( connectionDetails = NULL, targetDatabaseSchema, @@ -383,7 +308,7 @@ computeRechallengeFailCaseSeriesAnalyses <- function( outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow, show_subject_id = showSubjectId diff --git a/R/ExportingCsvFiles.R b/R/ExportingCsvFiles.R index bf0a094..08135cd 100644 --- a/R/ExportingCsvFiles.R +++ b/R/ExportingCsvFiles.R @@ -555,3 +555,76 @@ exportAttrition <- function( +exportCounts <- function( + executionPath, + outputFolder, + csvFilePrefix = 'c_', + minCellCount = 0 +){ + + # export target attrition + if(file.exists(file.path(executionPath, 'target_counts', 'result'))){ + andromeda <- Andromeda::loadAndromeda(file.path(executionPath, 'target_counts', 'result')) + + # censor + data <- andromeda$target_counts %>% dplyr::mutate( + nEvents = ifelse(.data$nEvents < !!minCellCount & .data$nEvents > 0, -1*minCellCount, .data$nEvents), + nPeople = ifelse(.data$nPeople < !!minCellCount & .data$nPeople > 0, -1*minCellCount, .data$nPeople) + ) %>% + dplyr::collect() + + colnames(data) <- SqlRender::camelCaseToSnakeCase(colnames(data)) + + # save the attrition + utils::write.csv( + x = data, + file = file.path(outputFolder, paste0(csvFilePrefix, 'target_counts', '.csv')), + row.names = FALSE + ) + } + + # export case attrition + if(file.exists(file.path(executionPath, 'case_counts', 'result'))){ + andromeda <- Andromeda::loadAndromeda(file.path(executionPath, 'case_counts', 'result')) + + # censor if the cases or non-cases are < min count + remove <- andromeda$case_counts %>% dplyr::filter( + .data$nEvents < !!minCellCount & .data$nEvents > 0, + .data$nPeople < !!minCellCount & .data$nPeople > 0 + ) %>% + dplyr::select("characterizationCaseId") %>% + dplyr::distinct() %>% + dplyr::mutate( + filter = TRUE + ) %>% + dplyr::collect() + + data <- andromeda$case_counts %>% + dplyr::left_join( + y = remove, + by = "characterizationCaseId", + copy = TRUE + ) %>% + dplyr::mutate(filter = dplyr::if_else(is.na(.data$filter), FALSE,.data$filter)) %>% + dplyr::mutate( + nEvents = ifelse(.data$filter, NA, .data$nEvents), + nPeople = ifelse(.data$filter, NA, .data$nPeople) + ) %>% + dplyr::select(-"filter") %>% + dplyr::collect() + + colnames(data) <- SqlRender::camelCaseToSnakeCase(colnames(data)) + + # save the attrition + utils::write.csv( + x = data, + file = file.path(outputFolder, paste0(csvFilePrefix, 'case_counts', '.csv')), + row.names = FALSE + ) + } + + return(invisible(TRUE)) +} + + + diff --git a/R/LookupCohortSettings.R b/R/LookupCohortSettings.R index 12f50f8..b7cd270 100644 --- a/R/LookupCohortSettings.R +++ b/R/LookupCohortSettings.R @@ -50,7 +50,8 @@ lookupCases <- function( riskWindowStart, endAnchor, riskWindowEnd, - minCaseSize # new + minCaseSize = 0, # new + applyMinSizeToNonCases = FALSE ){ sql <- " @@ -73,12 +74,16 @@ lookupCases <- function( ) cct ON lt.characterization_case_id = cct.characterization_case_id + +{@non_case_min}?{ INNER JOIN + (SELECT * FROM @lookup_schema.@case_count_table WHERE cohort_type = 'non-cases' AND n_people >= @min_case_size ) ncct ON lt.characterization_case_id = ncct.characterization_case_id +} WHERE lt.characterization_target_id in (@char_ids) AND lt.outcome_id in (@outcome_ids) @@ -102,7 +107,8 @@ lookupCases <- function( risk_window_start = riskWindowStart, end_anchor = endAnchor, risk_window_end = riskWindowEnd, - min_case_size = minCaseSize + min_case_size = minCaseSize, + non_case_min = applyMinSizeToNonCases ) sql <- SqlRender::translate( diff --git a/R/RiskFactorAnalysis.R b/R/RiskFactorAnalysis.R index 841e42b..0c01597 100644 --- a/R/RiskFactorAnalysis.R +++ b/R/RiskFactorAnalysis.R @@ -21,8 +21,6 @@ #' @param outcomeWashoutDays Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis #' @template timeAtRisk #' @param covariateSettings An object created using \code{FeatureExtraction::createCovariateSettings} -#' @param minTargetSize The minimum size of the target cohorts for them to have aggregate covariates calculated -#' @param minTwithOSize The minimum size of the cohorts corresponding to patients in the target with the outcome during time-at-risk for them to have aggregate covariates calculated #' #' @family Aggregate #' @return @@ -230,7 +228,8 @@ computeRiskFactorAnalyses <- function( riskWindowStart = settings$riskWindowStart, endAnchor = settings$endAnchor, riskWindowEnd = settings$riskWindowEnd, - minCaseSize = minCaseSize + minCaseSize = minCaseSize, + applyMinSizeToNonCases = TRUE ) # generate the targets, cases and non-cases ids diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 52f0dd2..42126e6 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -19,7 +19,11 @@ #' # example code #' #' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' @@ -129,13 +133,14 @@ addCharacterizationTargetIds <- function(settings){ # get the unique target + subsets studyPopulation <- unique(do.call('rbind', studyPopulationList)) %>% - dplyr::group_by(dplyr::across(-settingTypes)) %>% + dplyr::group_by(dplyr::across(-dplyr::all_of(settingTypes))) %>% dplyr::summarise( timeToEventSettings = any(.data$timeToEventSettings), dechallengeRechallengeSettings = any(.data$dechallengeRechallengeSettings), targetBaselineSettings = any(.data$targetBaselineSettings), riskFactorSettings = any(.data$riskFactorSettings), caseSeriesSettings = any(.data$caseSeriesSettings), + .groups = "drop" ) # give a new id called characterizationTargetIds per target and subset # characterizationTargetId always ends in 0 @@ -184,7 +189,11 @@ addCharacterizationTargetIds <- function(settings){ #' #' @examples #' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' @@ -229,7 +238,11 @@ saveCharacterizationSettings <- function( #' setPath <- file.path(tempdir(), 'charSet.json') #' #' drSet <- createDechallengeRechallengeSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' @@ -325,8 +338,8 @@ runCharacterizationAnalyses <- function( targetTable, outcomeDatabaseSchema, outcomeTable, - nestingCohortTable, - nestingCohortDatabaseSchema, + nestingCohortTable = targetTable, + nestingCohortDatabaseSchema = targetDatabaseSchema, outputDatabaseSchema = targetDatabaseSchema, outputTable = 'characterization_cohort', tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), @@ -542,7 +555,9 @@ runCharacterizationAnalyses <- function( mode = mode, minCharacterizationMean = minCharacterizationMean, minCovariateCount = minCovariateCount, - minSMD = minSMD + minSMD = minSMD, + minTargetSize = minTargetSize, + minCaseSize = minCaseSize ) # Now loop over the analysis jobs @@ -630,6 +645,12 @@ runCharacterizationAnalyses <- function( csvFilePrefix = csvFilePrefix, minCellCount = minCellCount ) + exportCounts( + executionPath = executionPath, + outputFolder = outputDirectory, + csvFilePrefix = csvFilePrefix, + minCellCount = minCellCount + ) invisible(outputDirectory) } @@ -761,7 +782,9 @@ exportSharedObjects <- function( mode, minCharacterizationMean, minCovariateCount, - minSMD + minSMD, + minTargetSize = minTargetSize, + minCaseSize = minCaseSize ){ # add code here to save execution_settings, @@ -871,17 +894,26 @@ exportSharedObjects <- function( targetDialect = attributes(connection)$dbms, tempEmulationSchema = tempEmulationSchema ) - data <- DatabaseConnector::querySql( + + andromeda <- Andromeda::andromeda() + + DatabaseConnector::querySqlToAndromeda( connection = connection, sql = sql, - snakeCaseToCamelCase = FALSE + andromeda = andromeda, + andromedaTableName = 'target_counts', + snakeCaseToCamelCase = TRUE ) - data$database_id <- databaseId - data$setting_id <- executionId - utils::write.csv( - x = formatDouble(data), - file = file.path(saveLocation, paste0(tablePrefix,'target_counts.csv')), - row.names = FALSE + + addDbAndSettings( + andromeda = andromeda, + databaseId = databaseId, + settingId = executionId + ) + + saveCharacterizationAndromeda( + andromeda = andromeda, + outputFolder = file.path(executionPath, 'target_counts') ) # extract case attrition @@ -953,17 +985,26 @@ exportSharedObjects <- function( targetDialect = attributes(connection)$dbms, tempEmulationSchema = tempEmulationSchema ) - data <- DatabaseConnector::querySql( + + andromeda <- Andromeda::andromeda() + + DatabaseConnector::querySqlToAndromeda( connection = connection, sql = sql, - snakeCaseToCamelCase = FALSE + andromeda = andromeda, + andromedaTableName = 'case_counts', + snakeCaseToCamelCase = TRUE ) - data$database_id <- databaseId - data$setting_id <- executionId - utils::write.csv( - x = formatDouble(data), - file = file.path(saveLocation, paste0(tablePrefix,'case_counts.csv')), - row.names = FALSE + + addDbAndSettings( + andromeda = andromeda, + databaseId = databaseId, + settingId = executionId + ) + + saveCharacterizationAndromeda( + andromeda = andromeda, + outputFolder = file.path(executionPath, 'case_counts') ) } @@ -977,7 +1018,9 @@ exportSharedObjects <- function( mode = mode, min_characterization_mean = minCharacterizationMean, min_covariate_count = minCovariateCount, - min_smd = minSMD + min_smd = minSMD, + min_target_size = minTargetSize, + min_case_size = minCaseSize ), file = file.path(saveLocation, paste0(tablePrefix,'execution_settings.csv')), row.names = FALSE diff --git a/R/StudyPopulation.R b/R/StudyPopulation.R index 83aba74..31ba619 100644 --- a/R/StudyPopulation.R +++ b/R/StudyPopulation.R @@ -18,7 +18,7 @@ #' for creating the study populations of interest #' @examples #' # Create study population settings with a washout period of 365 days and -#' restricted to adults for target dates that occur for the first time in 365 days. +#' # restricted to adults for target dates that occur for the first time in 365 days. #' populationSettings <- createStudyPopulationSettings( #' targetId = 1, #' limitToFirstInNDays = 365, diff --git a/R/TimeToEvent.R b/R/TimeToEvent.R index 1ddf39b..5890e05 100644 --- a/R/TimeToEvent.R +++ b/R/TimeToEvent.R @@ -69,46 +69,7 @@ createTimeToEventSettings <- function( return(result) } -#' Compute time to event study -#' -#' @template ConnectionDetails -#' @template TargetOutcomeTables -#' @template TempEmulationSchema -#' @param cdmDatabaseSchema The database schema containing the OMOP CDM data -#' @param settings The settings for the timeToEvent study -#' @param databaseId An identifier for the database (string) -#' @param outputFolder A directory to save the results as csv files -#' @param minCellCount The minimum cell value to display, values less than this will be replaced by -1 -#' @param progressBar Whether to display a progress bar while the analysis is running -#' @param executionId a unique id for the run -#' @param ... extra inputs -#' @family TimeToEvent -#' -#' @return -#' An \code{Andromeda::andromeda()} object containing the time to event results. -#' -#' @examples -#' # example code -#' -#' conDet <- exampleOmopConnectionDetails() -#' -#' tteSet <- createTimeToEventSettings( -#' targetIds = c(1,2), -#' outcomeIds = 3 -#' ) -#' -#' result <- computeTimeToEventAnalyses( -#' connectionDetails = conDet, -#' targetDatabaseSchema = 'main', -#' targetTable = 'cohort', -#' cdmDatabaseSchema = 'main', -#' settings = tteSet, -#' outputFolder = file.path(tempdir(), 'tte') -#' ) -#' -#' -#' -#' @export + computeTimeToEventAnalyses <- function( connectionDetails = NULL, targetDatabaseSchema, diff --git a/README.md b/README.md index 24be26b..6c9527e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,11 @@ targetIds <- c(1,2,4) ) dechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 63c5653..6e3fadc 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -89,6 +89,8 @@ execution_settings,mode,varchar(25),No,No,No,No,Whether Efficient/CohortIncidenc execution_settings,min_characterization_mean,float,No,No,No,No,The minimum fraction of patients who have a covariate for the covariate to be included in results execution_settings,min_covariate_count,int,No,No,No,No,The minimum number of patients who have a covariate for the covariate to be included in results (useful if cohorts are small) execution_settings,min_smd,float,No,No,No,No,The minimum standardized mean value a risk factor must have to be included in results +execution_settings,min_target_size,bigint,No,No,No,No,The minimum target cohort size to be included in target baseline, risk factor and case series results +execution_settings,min_case_size,bigint,No,No,No,No,The minimum case cohort size to be included in risk factor and case series results target_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier target_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier target_settings,characterization_target_id,bigint,Yes,Yes,No,No,The target cohort id after inclusion criteria used internally by characterization diff --git a/inst/sql/sql_server/RechallengeFailCaseSeries.sql b/inst/sql/sql_server/RechallengeFailCaseSeries.sql index 4b1c250..74dcfd1 100644 --- a/inst/sql/sql_server/RechallengeFailCaseSeries.sql +++ b/inst/sql/sql_server/RechallengeFailCaseSeries.sql @@ -6,7 +6,7 @@ IF OBJECT_ID('tempdb..#target_cohort', 'U') IS NOT NULL DROP TABLE #target_cohort; SELECT -ts.characterization_cohort_id as cohort_definition_id, +ts.characterization_target_id as cohort_definition_id, tc.subject_id, tc.cohort_start_date, tc.cohort_end_date, diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index e0b1371..5b87164 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -40,7 +40,7 @@ AND temp_cohort.cohort_start_date <= op.observation_period_end_date -- this is just to get the characterization_target_id INNER JOIN -(SELECT * FROM @target_settings_schema.@target_settings_table +(SELECT distinct * FROM @target_settings_schema.@target_settings_table WHERE limit_to_first_in_n_days = @limit_to_first_in_n_days AND min_prior_observation = @min_prior_observation -- added: diff --git a/man/computeDechallengeRechallengeAnalyses.Rd b/man/computeDechallengeRechallengeAnalyses.Rd deleted file mode 100644 index 8e4e4e1..0000000 --- a/man/computeDechallengeRechallengeAnalyses.Rd +++ /dev/null @@ -1,84 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DechallengeRechallenge.R -\name{computeDechallengeRechallengeAnalyses} -\alias{computeDechallengeRechallengeAnalyses} -\title{Compute dechallenge rechallenge study} -\usage{ -computeDechallengeRechallengeAnalyses( - connectionDetails = NULL, - targetDatabaseSchema, - targetTable, - outcomeDatabaseSchema = targetDatabaseSchema, - outcomeTable = targetTable, - tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), - settings, - databaseId = "database 1", - outputFolder, - minCellCount = 0, - progressBar = interactive(), - ... -) -} -\arguments{ -\item{connectionDetails}{An object of type `connectionDetails` as created using the -[DatabaseConnector::createConnectionDetails()] function.} - -\item{targetDatabaseSchema}{Schema name where your target cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{targetTable}{Name of the target cohort table.} - -\item{outcomeDatabaseSchema}{Schema name where your outcome cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{outcomeTable}{Name of the outcome cohort table.} - -\item{tempEmulationSchema}{Some database platforms like Oracle and Impala do not truly support temp tables. -To emulate temp tables, provide a schema with write privileges where temp tables -can be created} - -\item{settings}{The settings for the timeToEvent study} - -\item{databaseId}{An identifier for the database (string)} - -\item{outputFolder}{A directory to save the results as csv files} - -\item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} - -\item{progressBar}{Whether to display a progress bar while the analysis is running} - -\item{...}{extra inputs} -} -\value{ -An \code{Andromeda::andromeda()} object containing the dechallenge rechallenge results -} -\description{ -Compute dechallenge rechallenge study -} -\examples{ - -conDet <- exampleOmopConnectionDetails() - -drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), - outcomeIds = 3 -) - -computeDechallengeRechallengeAnalyses( - connectionDetails = conDet, - targetDatabaseSchema = 'main', - targetTable = 'cohort', - settings = drSet, - outputFolder = tempdir() -) - - -} -\seealso{ -Other DechallengeRechallenge: -\code{\link{computeRechallengeFailCaseSeriesAnalyses}()}, -\code{\link{createDechallengeRechallengeSettings}()} -} -\concept{DechallengeRechallenge} diff --git a/man/computeRechallengeFailCaseSeriesAnalyses.Rd b/man/computeRechallengeFailCaseSeriesAnalyses.Rd deleted file mode 100644 index 9daaa2f..0000000 --- a/man/computeRechallengeFailCaseSeriesAnalyses.Rd +++ /dev/null @@ -1,89 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/DechallengeRechallenge.R -\name{computeRechallengeFailCaseSeriesAnalyses} -\alias{computeRechallengeFailCaseSeriesAnalyses} -\title{Compute fine the subjects that fail the dechallenge rechallenge study} -\usage{ -computeRechallengeFailCaseSeriesAnalyses( - connectionDetails = NULL, - targetDatabaseSchema, - targetTable, - outcomeDatabaseSchema = targetDatabaseSchema, - outcomeTable = targetTable, - tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), - settings, - databaseId = "database 1", - showSubjectId = FALSE, - outputFolder, - minCellCount = 0, - progressBar = interactive(), - executionId, - ... -) -} -\arguments{ -\item{connectionDetails}{An object of type `connectionDetails` as created using the -[DatabaseConnector::createConnectionDetails()] function.} - -\item{targetDatabaseSchema}{Schema name where your target cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{targetTable}{Name of the target cohort table.} - -\item{outcomeDatabaseSchema}{Schema name where your outcome cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{outcomeTable}{Name of the outcome cohort table.} - -\item{tempEmulationSchema}{Some database platforms like Oracle and Impala do not truly support temp tables. -To emulate temp tables, provide a schema with write privileges where temp tables -can be created} - -\item{settings}{The settings for the timeToEvent study} - -\item{databaseId}{An identifier for the database (string)} - -\item{showSubjectId}{if F then subject_ids are hidden (recommended if sharing results)} - -\item{outputFolder}{A directory to save the results as csv files} - -\item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} - -\item{progressBar}{Whether to display a progress bar while the analysis is running} - -\item{executionId}{a unique id for the run} - -\item{...}{extra inputs} -} -\value{ -An \code{Andromeda::andromeda()} object with the case series details of the failed rechallenge -} -\description{ -Compute fine the subjects that fail the dechallenge rechallenge study -} -\examples{ - -conDet <- exampleOmopConnectionDetails() - -drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), - outcomeIds = 3 -) - -computeRechallengeFailCaseSeriesAnalyses( - connectionDetails = conDet, - targetDatabaseSchema = 'main', - targetTable = 'cohort', - settings = drSet, - outputFolder = tempdir() -) - -} -\seealso{ -Other DechallengeRechallenge: -\code{\link{computeDechallengeRechallengeAnalyses}()}, -\code{\link{createDechallengeRechallengeSettings}()} -} -\concept{DechallengeRechallenge} diff --git a/man/computeTimeToEventAnalyses.Rd b/man/computeTimeToEventAnalyses.Rd deleted file mode 100644 index 78e9e7a..0000000 --- a/man/computeTimeToEventAnalyses.Rd +++ /dev/null @@ -1,92 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/TimeToEvent.R -\name{computeTimeToEventAnalyses} -\alias{computeTimeToEventAnalyses} -\title{Compute time to event study} -\usage{ -computeTimeToEventAnalyses( - connectionDetails = NULL, - targetDatabaseSchema, - targetTable, - outcomeDatabaseSchema = targetDatabaseSchema, - outcomeTable = targetTable, - tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), - cdmDatabaseSchema, - settings, - databaseId = "database 1", - outputFolder, - minCellCount = 0, - progressBar = interactive(), - executionId, - ... -) -} -\arguments{ -\item{connectionDetails}{An object of type `connectionDetails` as created using the -[DatabaseConnector::createConnectionDetails()] function.} - -\item{targetDatabaseSchema}{Schema name where your target cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{targetTable}{Name of the target cohort table.} - -\item{outcomeDatabaseSchema}{Schema name where your outcome cohort table resides. Note that for SQL Server, -this should include both the database and schema name, for example -'scratch.dbo'.} - -\item{outcomeTable}{Name of the outcome cohort table.} - -\item{tempEmulationSchema}{Some database platforms like Oracle and Impala do not truly support temp tables. -To emulate temp tables, provide a schema with write privileges where temp tables -can be created} - -\item{cdmDatabaseSchema}{The database schema containing the OMOP CDM data} - -\item{settings}{The settings for the timeToEvent study} - -\item{databaseId}{An identifier for the database (string)} - -\item{outputFolder}{A directory to save the results as csv files} - -\item{minCellCount}{The minimum cell value to display, values less than this will be replaced by -1} - -\item{progressBar}{Whether to display a progress bar while the analysis is running} - -\item{executionId}{a unique id for the run} - -\item{...}{extra inputs} -} -\value{ -An \code{Andromeda::andromeda()} object containing the time to event results. -} -\description{ -Compute time to event study -} -\examples{ -# example code - -conDet <- exampleOmopConnectionDetails() - -tteSet <- createTimeToEventSettings( - targetIds = c(1,2), - outcomeIds = 3 -) - -result <- computeTimeToEventAnalyses( - connectionDetails = conDet, - targetDatabaseSchema = 'main', - targetTable = 'cohort', - cdmDatabaseSchema = 'main', - settings = tteSet, - outputFolder = file.path(tempdir(), 'tte') -) - - - -} -\seealso{ -Other TimeToEvent: -\code{\link{createTimeToEventSettings}()} -} -\concept{TimeToEvent} diff --git a/man/createCharacterizationSettings.Rd b/man/createCharacterizationSettings.Rd index b70f89a..669af3a 100644 --- a/man/createCharacterizationSettings.Rd +++ b/man/createCharacterizationSettings.Rd @@ -36,7 +36,11 @@ Specify one or more timeToEvent, dechallengeRechallenge and aggregateCovariate s # example code drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/man/createDechallengeRechallengeSettings.Rd b/man/createDechallengeRechallengeSettings.Rd index bc5388a..33c43c6 100644 --- a/man/createDechallengeRechallengeSettings.Rd +++ b/man/createDechallengeRechallengeSettings.Rd @@ -37,10 +37,5 @@ drSet <- createDechallengeRechallengeSettings( ) -} -\seealso{ -Other DechallengeRechallenge: -\code{\link{computeDechallengeRechallengeAnalyses}()}, -\code{\link{computeRechallengeFailCaseSeriesAnalyses}()} } \concept{DechallengeRechallenge} diff --git a/man/createRiskFactorSettings.Rd b/man/createRiskFactorSettings.Rd index 6a63a60..ef4ed33 100644 --- a/man/createRiskFactorSettings.Rd +++ b/man/createRiskFactorSettings.Rd @@ -26,9 +26,7 @@ createRiskFactorSettings( useProcedureOccurrenceShortTerm = TRUE, useMeasurementShortTerm = TRUE, useObservationShortTerm = TRUE, useDeviceExposureShortTerm = TRUE, useVisitConceptCountShortTerm = TRUE, endDays = 0, longTermStartDays = -365, - shortTermStartDays = -30), - minTargetSize = 0, - minTwithOSize = 0 + shortTermStartDays = -30) ) } \arguments{ @@ -49,10 +47,6 @@ or `"cohort end"`.} or `"cohort end"`.} \item{covariateSettings}{An object created using \code{FeatureExtraction::createCovariateSettings}} - -\item{minTargetSize}{The minimum size of the target cohorts for them to have aggregate covariates calculated} - -\item{minTwithOSize}{The minimum size of the cohorts corresponding to patients in the target with the outcome during time-at-risk for them to have aggregate covariates calculated} } \value{ A list with the settings diff --git a/man/createStudyPopulationSettings.Rd b/man/createStudyPopulationSettings.Rd index f225c3f..0688e41 100644 --- a/man/createStudyPopulationSettings.Rd +++ b/man/createStudyPopulationSettings.Rd @@ -46,7 +46,7 @@ create the study population settings } \examples{ # Create study population settings with a washout period of 365 days and - restricted to adults for target dates that occur for the first time in 365 days. +# restricted to adults for target dates that occur for the first time in 365 days. populationSettings <- createStudyPopulationSettings( targetId = 1, limitToFirstInNDays = 365, diff --git a/man/createTimeToEventSettings.Rd b/man/createTimeToEventSettings.Rd index f8c60ff..11053e8 100644 --- a/man/createTimeToEventSettings.Rd +++ b/man/createTimeToEventSettings.Rd @@ -30,9 +30,5 @@ tteSet <- createTimeToEventSettings( ) -} -\seealso{ -Other TimeToEvent: -\code{\link{computeTimeToEventAnalyses}()} } \concept{TimeToEvent} diff --git a/man/loadCharacterizationSettings.Rd b/man/loadCharacterizationSettings.Rd index 86b2a48..096a482 100644 --- a/man/loadCharacterizationSettings.Rd +++ b/man/loadCharacterizationSettings.Rd @@ -24,7 +24,11 @@ Input the directory containing the 'characterizationSettings.json' file and load setPath <- file.path(tempdir(), 'charSet.json') drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index ce8b068..2a580ce 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -10,8 +10,8 @@ runCharacterizationAnalyses( targetTable, outcomeDatabaseSchema, outcomeTable, - nestingCohortTable, - nestingCohortDatabaseSchema, + nestingCohortTable = targetTable, + nestingCohortDatabaseSchema = targetDatabaseSchema, outputDatabaseSchema = targetDatabaseSchema, outputTable = "characterization_cohort", tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"), @@ -30,7 +30,9 @@ runCharacterizationAnalyses( minCharacterizationMean = 0.001, minCovariateCount = 0, mode = "CohortIncidence", - minSMD = 0 + minSMD = 0, + minTargetSize = 0, + minCaseSize = 0 ) } \arguments{ @@ -91,6 +93,10 @@ can be created} \item{mode}{Select from Efficient (no exclusions to target based on washout)/CohortIncidence (excludes targets with outcome in washout if they have no time at risk)/PatientLevelPrediction (excludes targets with outcome during washout prior to index)} \item{minSMD}{The minimum standardized mean difference for the risk factor analysis} + +\item{minTargetSize}{The minimum target size to be included in targetBaseline, riskFactor or caseSeries} + +\item{minCaseSize}{The minimum case or non-case size to be included in riskFactor or caseSeries} } \value{ Multiple csv files in the outputDirectory. diff --git a/man/saveCharacterizationSettings.Rd b/man/saveCharacterizationSettings.Rd index c5f037b..1017994 100644 --- a/man/saveCharacterizationSettings.Rd +++ b/man/saveCharacterizationSettings.Rd @@ -22,7 +22,11 @@ Input the characterization settings and output a json file to a file named 'char } \examples{ drSet <- createDechallengeRechallengeSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 8714918..47b9179 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -6,3 +6,29 @@ withr::defer( }, testthat::teardown_env() ) + +skipIfCreateTargetCohortSqlUnavailable <- function() { + sqlAvailable <- !inherits( + try( + SqlRender::loadRenderTranslateSql( + sqlFilename = "CreateTargetCohortTable.sql", + packageName = "Characterization", + dbms = "sqlite", + tempEmulationSchema = "main", + characterization_schema = "main", + characterization_table = "char_table", + target_attrition_table = "target_attrition", + target_count_table = "target_count", + case_attrition_table = "case_attrition", + case_count_table = "case_count" + ), + silent = TRUE + ), + "try-error" + ) + + testthat::skip_if_not( + condition = sqlAvailable, + message = "CreateTargetCohortTable.sql not resolvable in this test context" + ) +} diff --git a/tests/testthat/test-CaseSeries.R b/tests/testthat/test-CaseSeries.R index c994174..c600b09 100644 --- a/tests/testthat/test-CaseSeries.R +++ b/tests/testthat/test-CaseSeries.R @@ -15,10 +15,12 @@ test_that("createCaseSeriesSettings", { ) res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -30,7 +32,7 @@ test_that("createCaseSeriesSettings", { ) testthat::expect_equal( - res$targetIds, + res$studyPopulationSettings$targetId, targetIds ) testthat::expect_equal( @@ -39,12 +41,12 @@ test_that("createCaseSeriesSettings", { ) testthat::expect_equal( - res$minPriorObservation, + unique(res$studyPopulationSettings$minPriorObservation), 10 ) testthat::expect_equal( - res$limitToFirstInNDays, + unique(res$studyPopulationSettings$limitToFirstInNDays), 9 ) @@ -91,10 +93,12 @@ test_that("error when using temporal features - risk factors", { testthat::expect_error( res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -111,10 +115,12 @@ test_that("error when using temporal features - risk factors", { testthat::expect_error( res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -137,10 +143,12 @@ test_that("createCaseSeriesSettings covariateList", { covariateSettings <- list(covariateSettings1, covariateSettings2) res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -150,7 +158,7 @@ test_that("createCaseSeriesSettings covariateList", { ) testthat::expect_equal( - res$targetIds, + res$studyPopulationSettings$targetId, targetIds ) testthat::expect_equal( @@ -172,10 +180,12 @@ test_that("getCaseSeriesJobs", { limitToFirstInNDays <- sample(300, 1) res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + minPriorObservation = minPriorObservation, + limitToFirstInNDays = limitToFirstInNDays + ), outcomeIds = outcomeIds, - minPriorObservation = minPriorObservation, - limitToFirstInNDays = limitToFirstInNDays, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -216,18 +226,28 @@ test_that("getCaseSeriesJobs", { testthat::expect_true(nrow(jobDf) == 2) # now check nTargetJobs = 3 + charSettings <- createCharacterizationSettings( + caseSeriesSettings = res + ) jobDf <- getCaseSeriesJobs( - characterizationSettings = createCharacterizationSettings( - caseSeriesSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 3 ) testthat::expect_true(nrow(jobDf) == 3) # check the target ids - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$targetIds == targetIds[1]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$targetIds == targetIds[2]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$targetIds == targetIds[3]) + tId1 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[1] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$characterizationTargetIds == tId1) + tId2 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[2] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$characterizationTargetIds == tId2) + tId3 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[3] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$characterizationTargetIds == tId3) # now check nTargetJobs = 4 jobDf <- getCaseSeriesJobs( @@ -250,6 +270,8 @@ test_that("getCaseSeriesJobs", { }) test_that("computeCaseSeriesAnalyses", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2, 4) outcomeIds <- c(3) covariateSettings <- createDuringCovariateSettings( @@ -258,10 +280,12 @@ test_that("computeCaseSeriesAnalyses", { ) res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 30 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 365, - minPriorObservation = 30, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -312,6 +336,8 @@ test_that("computeCaseSeriesAnalyses", { characterizationTable = tables$characterizationTable, # contains char cohorts targetSettingsTable = tables$targetSettingsTable, # contains map between settings and char cohort id caseSettingsTable = tables$caseSettingsTable, + caseCountTable = tables$caseCountTable, + minCaseSize = 0, tempEmulationSchema = 'main', settings = ParallelLogger::convertJsonToSettings(jobDf$settings[1]), databaseId = "madeup", @@ -376,14 +402,18 @@ test_that("computeCaseSeriesAnalyses", { # testing case series include/exclude covs test_that("testing during covs", { +skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2, 4) outcomeIds <- c(3) res <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 30 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 365, - minPriorObservation = 30, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', diff --git a/tests/testthat/test-ExportingCsvFiles.R b/tests/testthat/test-ExportingCsvFiles.R index 165250b..ec6fce1 100644 --- a/tests/testthat/test-ExportingCsvFiles.R +++ b/tests/testthat/test-ExportingCsvFiles.R @@ -732,8 +732,9 @@ test_that("exportAttrition", { # create example attrition andromeda <- Andromeda::andromeda() - andromeda$attrition <- data.frame( - cohortDefinitionId = c(10,20,30,40, 11,21,31,12,22,32), + andromeda$target_attrition <- data.frame( + characterizationCohortId = c(10,20,30,40, 11,21,31,12,22,32), + attrOrder = 1:10, attrReason = c('Target first in 365 - 365 prior obs', 'Target first in 365 - 365 prior obs', 'Target first in 365 - 365 prior obs', @@ -743,10 +744,14 @@ test_that("exportAttrition", { '3. Has outcome during TAR', '3. Has outcome during TAR' ), - n = c(1000,50,400,350, + nEvents = c(1000,50,400,350, 50,10,60, 50,10,60 ), + nPeople = c(1000,50,400,350, + 50,10,60, + 50,10,60 + ), databaseId = 'db', settingId = 'set1' ) @@ -754,44 +759,9 @@ test_that("exportAttrition", { # save to temp folder saveCharacterizationAndromeda( andromeda = andromeda, - outputFolder = file.path(tempFolder3,'attrition') + outputFolder = file.path(tempFolder3,'target_attrition') ) - # now create target and case settings - target_settings <- data.frame( - target_id = c(1,2,3,4), - limit_to_first_in_n_days = rep(365, 4), - min_prior_observation = rep(365, 4), - setting_id = 'set1', - characterization_target_id = c(10,20,30,40), - database_id = 'db' - ) - - case_settings <- data.frame( - outcome_id = rep(3,3), - outcome_washout_days = rep(90,3), - risk_window_start = rep(1,3), - start_anchor = rep('cohort_start',3), - risk_window_end = rep(365,3), - end_anchor = rep('cohort_start',3), - risk_factor_settings = rep(TRUE,3), - case_series_settings = rep(FALSE,3), - characterization_case_id = c(1,2,3), - setting_id = 'set1', - characterization_target_id = c(10,20,40), - database_id = 'db' - ) - - utils::write.csv( - x = target_settings, - file = file.path(tempFolder3, 'c_target_settings.csv') - ) - - utils::write.csv( - x = case_settings, - file = file.path(tempFolder3, 'c_case_settings.csv') - ) - exportAttrition( executionPath = tempFolder3, outputFolder = tempFolder3, @@ -800,11 +770,11 @@ exportAttrition( ) # load attrition -testthat::expect_true(file.exists(file.path(tempFolder3, 'c_attrition.csv'))) +testthat::expect_true(file.exists(file.path(tempFolder3, 'c_target_attrition.csv'))) -attrition <- utils::read.csv(file.path(tempFolder3, 'c_attrition.csv')) -testthat::expect_true(nrow(attrition) == 16) -testthat::expect_true(sum(colnames(attrition) %in% c('cohort_definition_id', 'attr_reason', 'n', 'database_id', 'setting_id')) == 5) +target_attrition <- utils::read.csv(file.path(tempFolder3, 'c_target_attrition.csv')) +testthat::expect_true(nrow(target_attrition) == 10) +testthat::expect_true(sum(colnames(target_attrition) %in% c('characterization_cohort_id','attr_order', 'attr_reason', 'n_people','n_events', 'database_id', 'setting_id')) == 7) # now test the minCellCount @@ -814,55 +784,23 @@ exportAttrition( csvFilePrefix = 'c_', minCellCount = 50 ) -attrition <- utils::read.csv(file.path(tempFolder3, 'c_attrition.csv')) -testthat::expect_true(sum(attrition$n < 50 & attrition$n != -50) == 0) -testthat::expect_true(sum(colnames(attrition) %in% c('cohort_definition_id', 'attr_reason', 'n', 'database_id', 'setting_id')) == 5) - -# now test csvFilePrefix -utils::write.csv( - x = target_settings, - file = file.path(tempFolder3, 'cccd_target_settings.csv') -) +target_attrition <- utils::read.csv(file.path(tempFolder3, 'c_target_attrition.csv')) +testthat::expect_true(sum(target_attrition$n_people < 50 & target_attrition$n_people != -50) == 0) +testthat::expect_true(sum(colnames(target_attrition) %in% c('characterization_cohort_id','attr_order', 'attr_reason', 'n_people','n_events', 'database_id', 'setting_id')) == 7) + -utils::write.csv( - x = case_settings, - file = file.path(tempFolder3, 'cccd_case_settings.csv') -) exportAttrition( executionPath = tempFolder3, outputFolder = tempFolder3, csvFilePrefix = 'cccd_', minCellCount = 12 ) -attrition <- utils::read.csv(file.path(tempFolder3, 'cccd_attrition.csv')) -testthat::expect_true(sum(attrition$n < 12 & attrition$n != -12) == 0) -testthat::expect_true(sum(colnames(attrition) %in% c('cohort_definition_id', 'attr_reason', 'n', 'database_id', 'setting_id')) == 5) +target_attrition <- utils::read.csv(file.path(tempFolder3, 'cccd_target_attrition.csv')) +testthat::expect_true(sum(target_attrition$n_people < 12 & target_attrition$n_people != -12) == 0) +testthat::expect_true(sum(colnames(target_attrition) %in% c('characterization_cohort_id','attr_order', 'attr_reason', 'n_people','n_events', 'database_id', 'setting_id')) == 7) -# test when no case_settings -utils::write.csv( - x = target_settings, - file = file.path(tempFolder3, 'c2_target_settings.csv') -) -return <- exportAttrition( - executionPath = tempFolder3, - outputFolder = tempFolder3, - csvFilePrefix = 'c2_', - minCellCount = 12 -) -attrition <- utils::read.csv(file.path(tempFolder3, 'c2_attrition.csv')) -testthat::expect_true(sum(attrition$n < 12 & attrition$n != -12) == 0) -testthat::expect_true(sum(colnames(attrition) %in% c('cohort_definition_id', 'attr_reason', 'n', 'database_id', 'setting_id')) == 5) -testthat::expect_true(nrow(attrition) == 4) +# TODO test case_attrition -# test no target or case settings -return <- exportAttrition( - executionPath = tempFolder3, - outputFolder = tempFolder3, - csvFilePrefix = 'c3_', - minCellCount = 12 -) -testthat::expect_false(return) -testthat::expect_true(!file.exists(file.path(tempFolder3, 'c3_attrition.csv'))) }) diff --git a/tests/testthat/test-RiskFactor.R b/tests/testthat/test-RiskFactor.R index fb90ae8..f851bab 100644 --- a/tests/testthat/test-RiskFactor.R +++ b/tests/testthat/test-RiskFactor.R @@ -19,10 +19,12 @@ test_that("createRiskFactorSettings", { ) res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -32,7 +34,7 @@ test_that("createRiskFactorSettings", { ) testthat::expect_equal( - res$targetIds, + res$studyPopulationSettings$targetId, targetIds ) testthat::expect_equal( @@ -41,12 +43,12 @@ test_that("createRiskFactorSettings", { ) testthat::expect_equal( - res$minPriorObservation, + unique(res$studyPopulationSettings$minPriorObservation), 10 ) testthat::expect_equal( - res$limitToFirstInNDays, + unique(res$studyPopulationSettings$limitToFirstInNDays), 9 ) @@ -84,10 +86,12 @@ test_that("error when using temporal features - risk factors", { testthat::expect_error( res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -104,10 +108,12 @@ test_that("error when using temporal features - risk factors", { testthat::expect_error( res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -132,10 +138,12 @@ test_that("createRiskFactorSettings covariateList", { covariateSettings <- list(covariateSettings1, covariateSettings2) res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 9, + minPriorObservation = 10 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 9, - minPriorObservation = 10, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -145,7 +153,7 @@ test_that("createRiskFactorSettings covariateList", { ) testthat::expect_equal( - res$targetIds, + res$studyPopulationSettings$targetId, targetIds ) testthat::expect_equal( @@ -168,10 +176,12 @@ test_that("getRiskFactorJobs", { limitToFirstInNDays <- sample(300, 1) res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + minPriorObservation = minPriorObservation, + limitToFirstInNDays = limitToFirstInNDays + ), outcomeIds = outcomeIds, - minPriorObservation = minPriorObservation, - limitToFirstInNDays = limitToFirstInNDays, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -215,18 +225,28 @@ test_that("getRiskFactorJobs", { testthat::expect_true(nrow(jobDf) == 2) # now check nTargetJobs = 3 + charSettings <- createCharacterizationSettings( + riskFactorSettings = res + ) jobDf <- getRiskFactorJobs( - characterizationSettings = createCharacterizationSettings( - riskFactorSettings = res - ), + characterizationSettings = charSettings, nTargetJobs = 3 ) testthat::expect_true(nrow(jobDf) == 3) # check the target ids - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$targetIds == targetIds[1]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$targetIds == targetIds[2]) - testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$targetIds == targetIds[3]) + tId1 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[1] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[1])$characterizationTargetIds == tId1) + tId2 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[2] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[2])$characterizationTargetIds == tId2) + tId3 <- charSettings$characterizationTargetLookup$characterizationTargetId[ + charSettings$characterizationTargetLookup$targetId == res$studyPopulationSettings$targetId[3] + ] + testthat::expect_true(ParallelLogger::convertJsonToSettings(jobDf$settings[3])$characterizationTargetIds == tId3) # now check nTargetJobs = 4 jobDf <- getRiskFactorJobs( @@ -249,6 +269,8 @@ test_that("getRiskFactorJobs", { }) test_that("computeRiskFactorAnalyses", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2, 4) outcomeIds <- c(3) covariateSettings <- FeatureExtraction::createCovariateSettings( @@ -258,10 +280,12 @@ test_that("computeRiskFactorAnalyses", { ) res <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 365, + minPriorObservation = 30 + ), outcomeIds = outcomeIds, - limitToFirstInNDays = 365, - minPriorObservation = 30, outcomeWashoutDays = 365, riskWindowStart = 1, startAnchor = 'cohort start', @@ -312,6 +336,8 @@ test_that("computeRiskFactorAnalyses", { characterizationTable = tables$characterizationTable, # contains char cohorts targetSettingsTable = tables$targetSettingsTable, # contains map between settings and char cohort id caseSettingsTable = tables$caseSettingsTable, + caseCountTable = , tables$caseCountTable, + minCaseSize = 0, tempEmulationSchema = 'main', settings = ParallelLogger::convertJsonToSettings(jobDf$settings[1]), databaseId = "madeup", diff --git a/tests/testthat/test-dbs.R b/tests/testthat/test-dbs.R index 0444539..1f6b879 100644 --- a/tests/testthat/test-dbs.R +++ b/tests/testthat/test-dbs.R @@ -181,38 +181,42 @@ for (dbmsPlatform in dbmsPlatforms) { targetIds <- c(1, 2, 4) outcomeIds <- c(3) + studyPop1 <- createStudyPopulationSettings(targetIds = 1) + studyPop2 <- createStudyPopulationSettings(targetIds = 2) + studyPopAll <- createStudyPopulationSettings(targetIds = targetIds) + timeToEventSettings1 <- createTimeToEventSettings( - targetIds = 1, + studyPopulationSettings = studyPop1, outcomeIds = c(3, 4) ) timeToEventSettings2 <- createTimeToEventSettings( - targetIds = 2, + studyPopulationSettings = studyPop2, outcomeIds = c(3, 4) ) dechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 ) targetBaselineSettings1 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE, useDemographicsAge = TRUE ) ) targetBaselineSettings2 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, covariateSettings = FeatureExtraction::createCovariateSettings( useConditionOccurrenceLongTerm = TRUE ) ) riskFactorSettings <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -224,7 +228,7 @@ for (dbmsPlatform in dbmsPlatforms) { ) caseSeriesSettings <- createCaseSeriesSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = "cohort start", @@ -258,6 +262,8 @@ for (dbmsPlatform in dbmsPlatforms) { targetTable = dbmsDetails$cohortTable, outcomeDatabaseSchema = dbmsDetails$cohortDatabaseSchema, outcomeTable = dbmsDetails$cohortTable, + nestingCohortTable = dbmsDetails$cohortTable, + nestingCohortDatabaseSchema = dbmsDetails$cohortDatabaseSchema, characterizationSettings = characterizationSettings, outputDirectory = file.path(tempFolder, "csv"), outputDatabaseSchema = dbmsDetails$cohortDatabaseSchema, diff --git a/tests/testthat/test-dechallengeRechallenge.R b/tests/testthat/test-dechallengeRechallenge.R index 3a70bac..da57330 100644 --- a/tests/testthat/test-dechallengeRechallenge.R +++ b/tests/testthat/test-dechallengeRechallenge.R @@ -51,6 +51,8 @@ test_that("createDechallengeRechallengeSettings", { }) test_that("computeDechallengeRechallengeAnalyses", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(2) outcomeIds <- c(3, 4) @@ -69,18 +71,41 @@ test_that("computeDechallengeRechallengeAnalyses", { dechallengeRechallengeSettings = drSet ) - charSet$dechallengeRechallengeSettings[[1]]$characterizationTargetIds <- 2 + jobDf <- getDechallengeRechallengeJobs( + characterizationSettings = charSet, + nTargetJobs = 1 + ) - # make the cohorts in a table dcLoc <- tempfile("runADechal") - dc <- Characterization::computeDechallengeRechallengeAnalyses( + tables <- generateCohorts( + characterizationSettings = charSet, + mode = 'PatientLevelPrediction', + incremental = FALSE, + executionPath = dcLoc, + connectionDetails = connectionDetails, + targetDatabaseSchema = "main", + targetTable = "cohort", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort", + outputDatabaseSchema = 'main', + outputTable = 'char_cohort', + cdmDatabaseSchema = "main", + tempEmulationSchema = "main", + progressBar = FALSE, + settingHash = 'set1', + dbHash = 'db1' + ) + + # make the cohorts in a table + + dc <- computeDechallengeRechallengeAnalyses( connectionDetails = connectionDetails, #targetDatabaseSchema = "main", #targetTable = "cohort", outcomeDatabaseSchema = "main", outcomeTable = "cohort", characterizationDatabaseSchema = "main", - characterizationTable = "cohort", + characterizationTable = tables$characterizationTable, settings = charSet$dechallengeRechallengeSettings[[1]], databaseId = "testing", outputFolder = dcLoc @@ -141,10 +166,40 @@ test_that("computeDechallengeRechallengeAnalyses", { camelCaseToSnakeCase = FALSE ) + DatabaseConnector::insertTable( + data = data.frame( + person_id = 1:4, + observation_period_start_date = rep(as.Date('1900-01-01'), 4), + observation_period_end_date = rep(as.Date('2100-01-01'), 4) + ), + connection = con, + databaseSchema = "main", + tableName = "observation_period", + createTable = TRUE, + dropTableIfExists = TRUE, + camelCaseToSnakeCase = FALSE + ) + + DatabaseConnector::insertTable( + data = data.frame( + person_id = 1:4, + year_of_birth = rep(1980, 4), + gender_concept_id = rep(0, 4) + ), + connection = con, + databaseSchema = "main", + tableName = "person", + createTable = TRUE, + dropTableIfExists = TRUE, + camelCaseToSnakeCase = FALSE + ) + DatabaseConnector::disconnect(con) drSet <- createDechallengeRechallengeSettings( - targetIds = 1, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 1 + ), outcomeIds = 2, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 30 @@ -154,10 +209,32 @@ test_that("computeDechallengeRechallengeAnalyses", { dechallengeRechallengeSettings = drSet ) - charSet$dechallengeRechallengeSettings[[1]]$characterizationTargetIds <- 1 - + jobDf <- getDechallengeRechallengeJobs( + characterizationSettings = charSet, + nTargetJobs = 1 + ) dcLoc <- tempfile("runADechal2") + + tables <- generateCohorts( + characterizationSettings = charSet, + mode = 'PatientLevelPrediction', + incremental = FALSE, + executionPath = dcLoc, + connectionDetails = connectionDetailsReal, + targetDatabaseSchema = "main", + targetTable = "cohort_dechal", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort_dechal", + outputDatabaseSchema = 'main', + outputTable = 'char_cohort', + cdmDatabaseSchema = "main", + tempEmulationSchema = "main", + progressBar = FALSE, + settingHash = 'set1', + dbHash = 'db1' + ) + dc <- computeDechallengeRechallengeAnalyses( connectionDetails = connectionDetailsReal, #targetDatabaseSchema = "main", @@ -165,7 +242,7 @@ test_that("computeDechallengeRechallengeAnalyses", { outcomeDatabaseSchema = "main", outcomeTable = "cohort_dechal", characterizationDatabaseSchema = "main", - characterizationTable = "cohort_dechal", + characterizationTable = tables$characterizationTable, settings = charSet$dechallengeRechallengeSettings[[1]], databaseId = "testing", outputFolder = dcLoc @@ -184,6 +261,8 @@ test_that("computeDechallengeRechallengeAnalyses", { }) test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { + skipIfCreateTargetCohortSqlUnavailable() + # check with made up date # subject 1 has 1 exposure for 30 days # subject 2 has 4 exposures for ~30 days with ~30 day gaps @@ -236,76 +315,84 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { dropTableIfExists = TRUE, camelCaseToSnakeCase = FALSE ) - DatabaseConnector::disconnect(con) - - drSet <- createDechallengeRechallengeSettings( - studyPopulationSettings = createStudyPopulationSettings( - targetIds = 1 - ), - outcomeIds = 2, - dechallengeStopInterval = 30, - dechallengeEvaluationWindow = 30 # 31 - ) - charSet <- createCharacterizationSettings( - dechallengeRechallengeSettings = drSet - ) - - # add the target_settings table - con <- DatabaseConnector::connect(connectionDetails = connectionDetailsReal) - DatabaseConnector::insertTable( data = data.frame( - characterizationTargetIds = c(10), - targetId = c(1) + person_id = 1:4, + observation_period_start_date = rep(as.Date('1900-01-01'), 4), + observation_period_end_date = rep(as.Date('2100-01-01'), 4) ), connection = con, databaseSchema = "main", - tableName = "target_settings", + tableName = "observation_period", createTable = TRUE, dropTableIfExists = TRUE, camelCaseToSnakeCase = FALSE ) - DatabaseConnector::disconnect(con) - - # add the characterization cohort table "characterization" - con <- DatabaseConnector::connect(connectionDetails = connectionDetailsReal) DatabaseConnector::insertTable( data = data.frame( - cohort_definition_id = rep(10, 10), - subject_id = c(1, 2, 2, 2, 2, 3, 3, 3, 4, 4), - cohort_start_date = as.Date(c( - "2001-01-01", - "2001-01-01", "2001-03-14", "2001-05-01", "2001-07-01", - "2001-01-01", "2001-03-01", "2001-05-01", - "2001-01-01", "2001-03-01" - )), - cohort_end_date = as.Date(c( - "2001-01-31", - "2001-01-31", "2001-03-16", "2001-05-30", "2001-07-31", - "2001-01-31", "2001-03-30", "2001-05-30", - "2001-01-31", "2001-03-30" - )) + person_id = 1:4, + year_of_birth = rep(1980, 4), + gender_concept_id = rep(0, 4) ), connection = con, databaseSchema = "main", - tableName = "characterization", + tableName = "person", createTable = TRUE, dropTableIfExists = TRUE, camelCaseToSnakeCase = FALSE ) DatabaseConnector::disconnect(con) + drSet <- createDechallengeRechallengeSettings( + studyPopulationSettings = createStudyPopulationSettings( + targetIds = 1 + ), + outcomeIds = 2, + dechallengeStopInterval = 30, + dechallengeEvaluationWindow = 30 # 31 + ) + charSet <- createCharacterizationSettings( + dechallengeRechallengeSettings = drSet + ) + + + jobDf <- getDechallengeRechallengeJobs( + characterizationSettings = charSet, + nTargetJobs = 1 + ) + dcLoc <- tempfile("runADechal2") + + tables <- generateCohorts( + characterizationSettings = charSet, + mode = 'PatientLevelPrediction', + incremental = FALSE, + executionPath = dcLoc, + connectionDetails = connectionDetailsReal, + targetDatabaseSchema = "main", + targetTable = "cohort", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort", + outputDatabaseSchema = 'main', + outputTable = 'char_cohort', + cdmDatabaseSchema = "main", + tempEmulationSchema = "main", + progressBar = FALSE, + settingHash = 'set1', + dbHash = 'db1' + ) + dc <- computeRechallengeFailCaseSeriesAnalyses( connectionDetails = connectionDetailsReal, targetDatabaseSchema = "main", targetTable = "cohort", + targetSettingsTable = tables$targetSettingsTable, settings = charSet$dechallengeRechallengeSettings[[1]], outcomeDatabaseSchema = "main", outcomeTable = "cohort", characterizationDatabaseSchema = "main", - characterizationTable = "characterization", + characterizationTable = tables$characterizationTable, databaseId = "testing", outputFolder = dcLoc ) @@ -319,13 +406,13 @@ test_that("computeRechallengeFailCaseSeriesAnalyses with known data", { testthat::expect_true(is.na(dc$subjectId)) dcLoc <- tempfile("runADechal3") - dc <- Characterization::computeRechallengeFailCaseSeriesAnalyses( + dc <- computeRechallengeFailCaseSeriesAnalyses( connectionDetails = connectionDetailsReal, targetDatabaseSchema = "main", targetTable = "cohort", + targetSettingsTable = tables$targetSettingsTable, characterizationDatabaseSchema = "main", - characterizationTable = "characterization", - targetSettingsTable = "target_settings", # new + characterizationTable = tables$characterizationTable, settings = charSet$dechallengeRechallengeSettings[[1]], outcomeDatabaseSchema = "main", outcomeTable = "cohort", diff --git a/tests/testthat/test-runCharacterization.R b/tests/testthat/test-runCharacterization.R index 7db6b0f..fb27845 100644 --- a/tests/testthat/test-runCharacterization.R +++ b/tests/testthat/test-runCharacterization.R @@ -129,34 +129,7 @@ test_that("runCharacterizationAnalyses", { length(characterizationSettings$caseSeriesSettings) == 1 ) - tempFile <- tempfile(fileext = ".json") - on.exit(unlink(tempFile)) - saveLoc <- saveCharacterizationSettings( - settings = characterizationSettings, - fileName = tempFile - ) - - testthat::expect_true(file.exists(tempFile)) - - loadedSettings <- loadCharacterizationSettings( - fileName = tempFile - ) - - # In R, empty arrays are automatically of type 'logical.' When loading JSON - # they are currently automatically of type 'list'. Neither is right or wrong, - # so ignoring distinction: - convertEmptyListToEmptyLogical <- function(object) { - if (is.list(object)) { - if (length(object) == 0) { - return(vector(mode = "logical", length = 0)) - } else { - return(lapply(object, convertEmptyListToEmptyLogical)) - } - } else { - return(object) - } - } - testthat::expect_equivalent(characterizationSettings, convertEmptyListToEmptyLogical(loadedSettings)) + skipIfCreateTargetCohortSqlUnavailable() tempFolder <- tempfile("Characterization") on.exit(unlink(tempFolder, recursive = TRUE), add = TRUE) @@ -168,6 +141,8 @@ test_that("runCharacterizationAnalyses", { targetTable = "cohort", outcomeDatabaseSchema = "main", outcomeTable = "cohort", + nestingCohortDatabaseSchema = 'main', + nestingCohortTable = "cohort", characterizationSettings = characterizationSettings, outputDatabaseSchema = 'main', @@ -236,18 +211,26 @@ test_that("runCharacterizationAnalyses", { file = file.path(tempFolder, "result", "c_time_to_event.csv"), show_col_types = FALSE ) + + charTids <- characterizationSettings$characterizationTargetLookup$characterizationTargetId[ + characterizationSettings$characterizationTargetLookup$targetId %in% c(1, 2) + ] + + testthat::expect_equivalent( - unique(tte$target_cohort_definition_id), - c(1, 2) + unique(tte$characterization_target_id), + unique(charTids) ) }) manualDataMin <- file.path(tempdir(), "manual_min.sqlite") -on.exit(file.remove(manualDataMin), add = TRUE) +on.exit(unlink(manualDataMin, force = TRUE), add = TRUE) test_that("min cell count works", { + skipIfCreateTargetCohortSqlUnavailable() + tempFolder <- tempfile("CharacterizationMin") on.exit(unlink(tempFolder, recursive = TRUE), add = TRUE) @@ -285,12 +268,10 @@ test_that("min cell count works", { obs_period <- data.frame( observation_period_id = 1:10, person_id = 1:10, - observation_period_start_date = rep("2000-12-31", 10), - observation_period_end_date = c("2000-12-31", rep("2020-12-31", 9)), + observation_period_start_date = rep(as.Date("2000-12-31"), 10), + observation_period_end_date = c(as.Date("2000-12-31"), rep(as.Date("2020-12-31"), 9)), period_type_concept_id = rep(1, 10) ) - obs_period$observation_period_start_date <- as.Date(obs_period$observation_period_start_date) - obs_period$observation_period_end_date <- as.Date(obs_period$observation_period_end_date) DatabaseConnector::insertTable( connection = con, databaseSchema = schema, @@ -388,18 +369,22 @@ test_that("min cell count works", { ) # create settings and run + minCellStudyPopulation <- createStudyPopulationSettings( + targetIds = 1, + minPriorObservation = 365 + ) + characterizationSettings <- createCharacterizationSettings( timeToEventSettings = createTimeToEventSettings( - targetIds = 1, + studyPopulationSettings = minCellStudyPopulation, outcomeIds = 2 ), dechallengeRechallengeSettings = createDechallengeRechallengeSettings( - targetIds = 1, + studyPopulationSettings = minCellStudyPopulation, outcomeIds = 2 ), targetBaselineSettings = createTargetBaselineSettings( - targetIds = 1, - minPriorObservation = 365, + studyPopulationSettings = minCellStudyPopulation, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsAge = TRUE, useDemographicsGender = TRUE, @@ -415,6 +400,8 @@ test_that("min cell count works", { targetTable = "cohort", outcomeDatabaseSchema = "main", outcomeTable = "cohort", + nestingCohortDatabaseSchema = 'main', + nestingCohortTable = "cohort", characterizationSettings = characterizationSettings, outputDirectory = file.path(tempFolder, "result_mincell"), executionPath = file.path(tempFolder, "execution_mincell"), diff --git a/tests/testthat/test-targetAnalysis.R b/tests/testthat/test-targetAnalysis.R index 925c9b5..85dd29e 100644 --- a/tests/testthat/test-targetAnalysis.R +++ b/tests/testthat/test-targetAnalysis.R @@ -220,6 +220,8 @@ test_that("getTargetBaselineJobs", { }) test_that("computeTargetBaselineAnalyses", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2, 4) covariateSettings <- FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE, @@ -271,6 +273,8 @@ test_that("computeTargetBaselineAnalyses", { cdmVersion = 5, targetDatabaseSchema = "main", targetTable = "cohort", + targetCountTable = tables$targetCountTable, + minTargetSize = 0, characterizationDatabaseSchema = 'main', characterizationTable = tables$characterizationTable, # contains char cohorts diff --git a/tests/testthat/test-timeToEvent.R b/tests/testthat/test-timeToEvent.R index d9111c6..cf3a9be 100644 --- a/tests/testthat/test-timeToEvent.R +++ b/tests/testthat/test-timeToEvent.R @@ -5,12 +5,14 @@ test_that("createTimeToEventSettings", { outcomeIds <- sample(x = 100, size = sample(10, 1)) res <- createTimeToEventSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds ) testthat::expect_true( - length(unique(res$targetIds)) == length(targetIds) + length(unique(res$studyPopulationSettings$targetId)) == length(targetIds) ) testthat::expect_true( @@ -19,22 +21,58 @@ test_that("createTimeToEventSettings", { }) test_that("computeTimeToEventSettings", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2) outcomeIds <- c(3, 4) res <- createTimeToEventSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds + ), outcomeIds = outcomeIds ) + characterizationSettings <- createCharacterizationSettings( + timeToEventSettings = res + ) + + jobDf <- getTimeToEventJobs( + characterizationSettings = characterizationSettings, + nTargetJobs = 1 + ) + tteFolder <- tempfile("tte") + tables <- generateCohorts( + characterizationSettings = characterizationSettings, + mode = 'PatientLevelPrediction', + incremental = FALSE, + executionPath = tteFolder, + connectionDetails = connectionDetails, + targetDatabaseSchema = "main", + targetTable = "cohort", + outcomeDatabaseSchema = "main", + outcomeTable = "cohort", + outputDatabaseSchema = 'main', + outputTable = 'char_cohort', + cdmDatabaseSchema = "main", + tempEmulationSchema = "main", + progressBar = FALSE, + settingHash = 'set1', + dbHash = 'db1' + ) + computeTimeToEventAnalyses( connectionDetails = connectionDetails, cdmDatabaseSchema = "main", targetDatabaseSchema = "main", targetTable = "cohort", - settings = res, + outcomeDatabaseSchema = "main", + outcomeTable = "cohort", + characterizationDatabaseSchema = 'main', + characterizationTable = tables$characterizationTable, + settings = ParallelLogger::convertJsonToSettings(jobDf$settings[1]), outputFolder = tteFolder, databaseId = "tte_test" ) @@ -55,14 +93,18 @@ test_that("computeTimeToEventSettings", { ) ) <= length(targetIds) ) + + charTargetIds <- characterizationSettings$characterizationTargetLookup$characterizationTargetId[ + characterizationSettings$characterizationTargetLookup$targetId %in% targetIds + ] + testthat::expect_true( sum(unique( - tte$targetCohortDefinitionId - ) %in% targetIds) == - length(unique(tte$targetCohortDefinitionId)) + tte$characterizationTargetId + ) %in% charTargetIds) == + length(unique(tte$characterizationTargetId)) ) - testthat::expect_true( length( unique( diff --git a/vignettes/Specification.Rmd b/vignettes/Specification.Rmd index 75aa9ef..27c7317 100644 --- a/vignettes/Specification.Rmd +++ b/vignettes/Specification.Rmd @@ -25,7 +25,7 @@ vignette: > ## Inputs -A vector of targetIds and a vector of outcomeIds +A studyPopulationSettings object (containing targetIds and any target population restrictions) and a vector of outcomeIds. ## Output @@ -206,7 +206,7 @@ knitr::kable( ## Inputs -A vector of targetIds, a vector of outcomeIds, an integer dechallengeStopInterval and an integer dechallengeEvaluationWindow. +A studyPopulationSettings object (containing targetIds and any target population restrictions), a vector of outcomeIds, an integer dechallengeStopInterval and an integer dechallengeEvaluationWindow. ## Output @@ -336,7 +336,7 @@ knitr::kable( ## Inputs -A vector of targetIds plus the minimum prior observation required for the target cohorts and minimum time before target exposures and specifying which features to extract (covariateSettings). +A studyPopulationSettings object (containing targetIds plus target population restrictions such as minimum prior observation and first-exposure limits) and covariateSettings specifying which features to extract. ## Outputs @@ -360,9 +360,11 @@ covariateSettings <- FeatureExtraction::createCovariateSettings( ) targetSettings <- Characterization::createTargetBaselineSettings( - targetIds = c(1,2), - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, + studyPopulationSettings = Characterization::createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = limitToFirstInNDays, + minPriorObservation = minPriorObservation + ), covariateSettings = covariateSettings ) @@ -425,7 +427,7 @@ This analysis lets users compare the mean values of the features between databas ## Inputs -A vector of targetIds and outcomeIds plus the minimum prior observation required for the target cohorts, the outcome washout days for the outcomes, settings for the time-at-risk and covariate settings specifying which features to extract. +A studyPopulationSettings object (containing targetIds plus target population restrictions), outcomeIds, outcome washout days, time-at-risk settings, and covariate settings specifying which features to extract. ## Outputs @@ -454,10 +456,12 @@ covariateSettings <- FeatureExtraction::createCovariateSettings( ) rfSettings <- Characterization::createRiskFactorSettings( - targetIds = targetId, + studyPopulationSettings = Characterization::createStudyPopulationSettings( + targetIds = targetId, + limitToFirstInNDays = limitToFirstInNDays, + minPriorObservation = minPriorObservation + ), outcomeIds = outcomeId, - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, outcomeWashoutDays = outcomeWashoutDays, riskWindowStart = riskWindowStart, startAnchor = startAnchor, @@ -619,7 +623,7 @@ The cases series looks at the patients in a target cohort who have the outcome d ## Inputs -A vector of targetIds and outcomeIds plus the minimum prior observation required for the target cohorts, the outcome washout days for the outcomes, settings for the time-at-risk and covariate settings specifying which features to extract. +A studyPopulationSettings object (containing targetIds plus target population restrictions), outcomeIds, outcome washout days, time-at-risk settings, and covariate settings specifying which features to extract. In addition you need to specify how long before target index to extract before index features (preTargetIndexDays) and how long after outcome index to extract after index features (postOutcomeIndexDays). @@ -659,10 +663,12 @@ caseCovariateSettings <- Characterization::createDuringCovariateSettings( ) caseSeriesSettings <- Characterization::createCaseSeriesSettings( - targetIds = targetId, + studyPopulationSettings = Characterization::createStudyPopulationSettings( + targetIds = targetId, + limitToFirstInNDays = limitToFirstInNDays, + minPriorObservation = minPriorObservation + ), outcomeIds = outcomeId, - limitToFirstInNDays = limitToFirstInNDays, - minPriorObservation = minPriorObservation, outcomeWashoutDays = outcomeWashoutDays, riskWindowStart = riskWindowStart, startAnchor = startAnchor, diff --git a/vignettes/UsingPackage.Rmd b/vignettes/UsingPackage.Rmd index b1f47bb..1edf0fb 100644 --- a/vignettes/UsingPackage.Rmd +++ b/vignettes/UsingPackage.Rmd @@ -54,9 +54,7 @@ connectionDetails <- Characterization::exampleOmopConnectionDetails() To run an 'Target Baseline Covariate' analysis you need to create a setting object using `createTargetBaselineSettings`. This requires specifying: -- one or more targetIds (these must be pre-generated in a cohort table) -- a limitToFirstInNDays that removes target exposures that occur within this number of days of a prior exposure. Use 99999 to restrict to first target exposure. -- a minPriorObservation that specifies the minimum number of days in the database a person needs to have at target index to be included. +- studyPopulationSettings created using `createStudyPopulationSettings` to define targetIds and population restrictions. - the covariate settings using `FeatureExtraction::createCovariateSettings` or by creating your own custom feature extraction code. Using the Eunomia data were we previous generated four cohorts, we can use cohort ids 1,2 and 4 as the targetIds: @@ -79,9 +77,11 @@ If we want to create the aggregate features for all our target cohort restricted ```{r eval=TRUE} exampleTargetBaselineSettings <- createTargetBaselineSettings( - targetIds = exampleTargetIds, - limitToFirstInNDays = 99999, - minPriorObservation = 365, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = exampleTargetIds, + limitToFirstInNDays = 99999, + minPriorObservation = 365 + ), covariateSettings = exampleCovariateSettings ) ``` @@ -118,10 +118,8 @@ You can then see the results in the location `file.path(tempdir(), 'example_char To run an 'Risk Factor Covariate' analysis you need to create a setting object using `createRiskFactorSettings`. This requires specifying: -- one or more targetIds (these must be pre-generated in a cohort table) +- studyPopulationSettings created using `createStudyPopulationSettings` to define targetIds and population restrictions. - one or more outcomeIds (these must be pre-generated in a cohort table) -- a limitToFirstInNDays that removes target exposures that occur within this number of days of a prior exposure. Use 99999 to restrict to first target exposure. -- a minPriorObservation that specifies the minimum number of days in the database a person needs to have at target index to be included. - the covariate settings using `FeatureExtraction::createCovariateSettings` or by creating your own custom feature extraction code. - the time-at-risk settings + riskWindowStart @@ -150,13 +148,15 @@ If we want to create the aggregate features for all our cases/non-cases which ar ```{r eval=TRUE} exampleRiskFactorSettings <- createRiskFactorSettings( - targetIds = exampleTargetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = exampleTargetIds, + limitToFirstInNDays = 99999, # limit to first target exposure + minPriorObservation = 365 + ), outcomeIds = exampleOutcomeIds, - limitToFirstInNDays = 99999, # limit to first target exposure riskWindowStart = 1, startAnchor = "cohort start", riskWindowEnd = 365, endAnchor = "cohort start", outcomeWashoutDays = 9999, - minPriorObservation = 365, covariateSettings = exampleCovariateSettings ) ``` @@ -202,10 +202,8 @@ You can then see the results in the location `file.path(tempdir(), 'example_char To run an 'Case Series Covariate' analysis you need to create a setting object using `createCaseSeriesSettings`. This requires specifying: -- one or more targetIds (these must be pre-generated in a cohort table) +- studyPopulationSettings created using `createStudyPopulationSettings` to define targetIds and population restrictions. - one or more outcomeIds (these must be pre-generated in a cohort table) -- a limitToFirstInNDays that removes target exposures that occur within this number of days of a prior exposure. Use 99999 to restrict to first target exposure. -- a minPriorObservation that specifies the minimum number of days in the database a person needs to have at target index to be included. - the case covariate settings using `Characterization::createDuringCovariateSettings` or by creating your own custom feature extraction code. - the time-at-risk settings + riskWindowStart @@ -235,13 +233,15 @@ We also need to specify two variables `casePreTargetDuration` which is the numbe ```{r eval=TRUE} exampleCaseSeriesSettings <- createCaseSeriesSettings( - targetIds = exampleTargetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = exampleTargetIds, + limitToFirstInNDays = 99999, # limit to first target index + minPriorObservation = 365 + ), outcomeIds = exampleOutcomeIds, - limitToFirstInNDays = 99999, # limit to first target index riskWindowStart = 1, startAnchor = "cohort start", riskWindowEnd = 365, endAnchor = "cohort start", outcomeWashoutDays = 9999, - minPriorObservation = 365, caseCovariateSettings = exampleCaseCovariateSettings, casePreTargetDuration = 90, casePostOutcomeDuration = 90 @@ -282,7 +282,7 @@ You can then see the results in the location `file.path(tempdir(), 'example_char To run a 'Dechallenge Rechallenge' analysis you need to create a setting object using `createDechallengeRechallengeSettings`. This requires specifying: -- one or more targetIds (these must be pre-generated in a cohort table) +- studyPopulationSettings created using `createStudyPopulationSettings` to define targetIds and population restrictions. - one or more outcomeIds (these must be pre-generated in a cohort table) - dechallengeStopInterval - dechallengeEvaluationWindow @@ -298,7 +298,9 @@ If we want to create the dechallenge rechallenge for all our target cohorts and ```{r eval=TRUE} exampleDechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = exampleTargetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = exampleTargetIds + ), outcomeIds = exampleOutcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 @@ -339,12 +341,14 @@ failed <- computeRechallengeFailCaseSeriesAnalyses( To run a 'Time-to-event' analysis you need to create a setting object using `createTimeToEventSettings`. This requires specifying: -- one or more targetIds (these must be pre-generated in a cohort table) +- studyPopulationSettings created using `createStudyPopulationSettings` to define targetIds and population restrictions. - one or more outcomeIds (these must be pre-generated in a cohort table) ```{r eval=TRUE} exampleTimeToEventSettings <- createTimeToEventSettings( - targetIds = exampleTargetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = exampleTargetIds + ), outcomeIds = exampleOutcomeIds ) ``` From 0e879bd7203b5078f3e11f6a3af104cff71d5d29 Mon Sep 17 00:00:00 2001 From: jreps Date: Mon, 15 Jun 2026 16:14:06 -0400 Subject: [PATCH 05/39] updating results tables via migrations fixing result table creation and uploading for new result tables --- DESCRIPTION | 2 +- R/ViewShiny.R | 26 +--- .../resultsDataModelSpecification.csv | 18 +-- inst/sql/sql_server/ResultTables.sql | 12 +- .../Migration_2-v3_1_0_table_change.sql | 147 ++++++++++++++++++ tests/testthat/test-manualData.R | 32 ++-- tests/testthat/test-viewShiny.R | 18 ++- 7 files changed, 193 insertions(+), 62 deletions(-) create mode 100644 inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql diff --git a/DESCRIPTION b/DESCRIPTION index 7e55026..170bb88 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 3.0.1 +Version: 3.1.0 Date: 2026-4-15 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 8fea895..69ff1f0 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -155,33 +155,13 @@ prepareCharacterizationShiny <- function( connection = con, sql = paste0("select distinct TARGET_ID from ", tablePrefix, csvTablePrefix, "target_settings;"), snakeCaseToCamelCase = TRUE - )$targetCohortId, + )$targetId, DatabaseConnector::querySql( connection = con, sql = paste0("select distinct OUTCOME_ID from ", tablePrefix, csvTablePrefix, "case_settings;"), snakeCaseToCamelCase = TRUE - )$outcomeCohortId, - DatabaseConnector::querySql( - connection = con, - sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), - snakeCaseToCamelCase = TRUE - )$targetCohortDefinitionId, - DatabaseConnector::querySql( - connection = con, - sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), - snakeCaseToCamelCase = TRUE - )$outcomeCohortDefinitionId, - DatabaseConnector::querySql( - connection = con, - sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), - snakeCaseToCamelCase = TRUE - )$targetCohortDefinitionId, - DatabaseConnector::querySql( - connection = con, - sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), - snakeCaseToCamelCase = TRUE - )$outcomeCohortDefinitionId - ) + )$outcomeId + ) ) diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index 6e3fadc..b0a9b77 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -28,7 +28,7 @@ rechallenge_fail_case_series,rechallenge_outcome_start_date_offset,int,Yes,No,No dechallenge_rechallenge,database_id,varchar(100),Yes,Yes,No,No,The database identifier dechallenge_rechallenge,dechallenge_stop_interval,int,Yes,Yes,No,No,The dechallenge stop interval dechallenge_rechallenge,dechallenge_evaluation_window,int,Yes,Yes,No,No,The dechallenge evaluation window -dechallenge_rechallenge,characterization_cohort_id,bigint,Yes,Yes,No,No,The characterization cohort definition id for the target cohort +dechallenge_rechallenge,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort definition id for the target cohort dechallenge_rechallenge,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort dechallenge_rechallenge,num_exposure_eras,int,Yes,No,No,No,The number of exposure eras dechallenge_rechallenge,num_persons_exposed,int,Yes,No,No,No,The number of persons exposed @@ -84,19 +84,19 @@ target_covariates_continuous,p_75_value,float,No,No,No,No,The 75th percentile target_covariates_continuous,p_90_value,float,No,No,No,No,The 90th percentile execution_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier execution_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier -execution_settings,database_hash,varchar(50),Yes,No,No,No, +execution_settings,database_hash,varchar(50),Yes,No,No,No,The hash of the database identifier execution_settings,mode,varchar(25),No,No,No,No,Whether Efficient/CohortIncidence/PatientLevelPrediction mode was used for risk factor non-cases execution_settings,min_characterization_mean,float,No,No,No,No,The minimum fraction of patients who have a covariate for the covariate to be included in results execution_settings,min_covariate_count,int,No,No,No,No,The minimum number of patients who have a covariate for the covariate to be included in results (useful if cohorts are small) execution_settings,min_smd,float,No,No,No,No,The minimum standardized mean value a risk factor must have to be included in results -execution_settings,min_target_size,bigint,No,No,No,No,The minimum target cohort size to be included in target baseline, risk factor and case series results +execution_settings,min_target_size,bigint,No,No,No,No,"The minimum target cohort size to be included in target baseline, risk factor and case series results" execution_settings,min_case_size,bigint,No,No,No,No,The minimum case cohort size to be included in risk factor and case series results target_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier target_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier target_settings,characterization_target_id,bigint,Yes,Yes,No,No,The target cohort id after inclusion criteria used internally by characterization target_settings,target_id,bigint,No,No,No,No,The target cohort id -target_settings,limit_to_first_in_n_days,int,No,No,No,No,Target exposures are only included if they occur >= first_in_n_days days after the last exposure -target_settings,min_prior_observation,int,No,No,No,No,Target exposures with < min_prior_obs days observation before exposure are excluded +target_settings,limit_to_first_in_n_days,int,No,No,No,No,"Target exposures are only included if they occur >= first_in_n_days days after the last exposure" +target_settings,min_prior_observation,int,No,No,No,No,"Target exposures with < min_prior_obs days observation before exposure are excluded" target_settings,nesting_cohort_id,bigint,No,No,No,No,The nesting id for the popualtion of interest target_settings,min_age,int,No,No,No,No,The min age to be includedfor the popualtion of interest target_settings,max_age,int,No,No,No,No,The max age to be includedfor the popualtion of interest @@ -113,7 +113,7 @@ case_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier case_settings,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" case_settings,characterization_target_id,bigint,Yes,No,No,No,The target cohort id after inclusion criteria used internally by characterization case_settings,outcome_id,bigint,No,No,No,No,The outcome cohort id -case_settings,outcome_washout_days,int,No,No,No,No,Outcome exposures with < outcome_washout_days days after the last outcome exposure are excluded +case_settings,outcome_washout_days,int,No,No,No,No,"Outcome exposures with < outcome_washout_days days after the last outcome exposure are excluded" case_settings,start_anchor,varchar(15),No,No,No,No,The start anchor case_settings,end_anchor,varchar(15),No,No,No,No,The end anchor case_settings,risk_window_start,int,No,No,No,No,The risk window start @@ -204,9 +204,9 @@ target_counts,characterization_target_id,bigint,Yes,Yes,No,No,The characterizati target_counts,n_events,bigint,No,No,No,No,The number of events target_counts,n_people,bigint,No,No,No,No,The number of people target_counts,database_id,varchar(100),Yes,Yes,No,No,The database identifier -target_counts,setting_id,setting_id,varchar(50),Yes,Yes,No,No,The run identifier -case_counts,characterization_case_id -case_counts,cohort_type,varchar(50),No,No,No,No,Whether the count is a case or non-case +target_counts,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +case_counts,characterization_case_id,bigint,Yes,Yes,No,No,The characterization case id +case_counts,cohort_type,varchar(50),No,Yes,No,No,Whether the count is a case or non-case case_counts,n_events,bigint,No,No,No,No,The number of events case_counts,n_people,bigint,No,No,No,No,The number of people case_counts,database_id,varchar(100),Yes,Yes,No,No,The database identifier diff --git a/inst/sql/sql_server/ResultTables.sql b/inst/sql/sql_server/ResultTables.sql index 2688468..47190ad 100644 --- a/inst/sql/sql_server/ResultTables.sql +++ b/inst/sql/sql_server/ResultTables.sql @@ -219,6 +219,8 @@ CREATE TABLE @my_schema.@table_prefixexecution_settings ( min_characterization_mean FLOAT, min_covariate_count INT, min_smd FLOAT, + min_target_size BIGINT, + min_case_size BIGINT, PRIMARY KEY (setting_id, database_id) ); @@ -253,13 +255,3 @@ CREATE TABLE @my_schema.@table_prefixcase_series_settings ( case_post_outcome_duration int, PRIMARY KEY (setting_id) ); - --- added this table -CREATE TABLE @my_schema.@table_prefixattrition ( - database_id varchar(100) NOT NULL, - setting_id varchar(50) NOT NULL, - cohort_definition_id BIGINT, - attr_reason VARCHAR(200), - n BIGINT, - PRIMARY KEY (setting_id, database_id, cohort_definition_id, attr_reason) -); diff --git a/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql b/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql new file mode 100644 index 0000000..b7105ca --- /dev/null +++ b/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql @@ -0,0 +1,147 @@ +-- Database migrations for verion 0.3.0 +-- This migration updates the schema: + -- 1. to store the charcterization version + -- 2. Add a migrations table for supporting database migrations + +{DEFAULT @package_version = package_version} +{DEFAULT @migration = migration} +{DEFAULT @table_prefix = ''} + + + +-- =========================== +-- 1) Create target_attrition table +-- =========================== +DROP TABLE IF EXISTS @database_schema.@table_prefixtarget_attrition; + +--HINT DISTRIBUTE ON RANDOM +CREATE TABLE @database_schema.@table_prefixtarget_attrition( + characterization_target_id BIGINT, + attr_order INT, + attr_reason VARCHAR(100), + n_events BIGINT, + n_people BIGINT, + database_id VARCHAR(100), + setting_id VARCHAR(50), + PRIMARY KEY (setting_id, database_id, characterization_target_id, attr_order) +); +-- =========================== + + +-- =========================== +-- 2) Create case_attrition table +-- =========================== +DROP TABLE IF EXISTS @database_schema.@table_prefixcase_attrition; + +--HINT DISTRIBUTE ON RANDOM +CREATE TABLE @database_schema.@table_prefixcase_attrition( + characterization_case_id BIGINT, + attr_order INT, + attr_reason VARCHAR(100), + n_events BIGINT, + n_people BIGINT, + database_id VARCHAR(100), + setting_id VARCHAR(50), + PRIMARY KEY (setting_id, database_id, characterization_case_id, attr_order) +); +-- =========================== + + + + +-- =========================== +-- 3) Create target_count table +-- =========================== +DROP TABLE IF EXISTS @database_schema.@table_prefixtarget_counts; + +--HINT DISTRIBUTE ON RANDOM +CREATE TABLE @database_schema.@table_prefixtarget_counts( + characterization_target_id BIGINT, + n_events BIGINT, + n_people BIGINT, + database_id VARCHAR(100), + setting_id VARCHAR(50), + PRIMARY KEY (setting_id, database_id, characterization_target_id) +); +-- =========================== + + +-- =========================== +-- 4) Create case_count table +-- =========================== +DROP TABLE IF EXISTS @database_schema.@table_prefixcase_counts; + +--HINT DISTRIBUTE ON RANDOM +CREATE TABLE @database_schema.@table_prefixcase_counts( + characterization_case_id BIGINT, + cohort_type VARCHAR(50), + n_events BIGINT, + n_people BIGINT, + database_id VARCHAR(100), + setting_id VARCHAR(50), + PRIMARY KEY (setting_id, database_id, characterization_case_id, cohort_type) +); +-- =========================== + + +-- =========================== +-- 5) Rename target_cohort_definition_id to characterization_target_id +-- =========================== +-- dechallenge_rechallenge/rechallenge_fail_case_series/time_to_event +-- Change to target_cohort_definition_id characterization_target_id + +ALTER TABLE @database_schema.@table_prefixdechallenge_rechallenge +RENAME COLUMN target_cohort_definition_id to characterization_target_id; + +ALTER TABLE @database_schema.@table_prefixrechallenge_fail_case_series +RENAME COLUMN target_cohort_definition_id to characterization_target_id; + +ALTER TABLE @database_schema.@table_prefixtime_to_event +RENAME COLUMN target_cohort_definition_id to characterization_target_id; + + +-- =========================== +-- 6) Add columns in target_settings +-- =========================== +-- target_settings: add +-- nesting_cohort_id bigint / min_age int / max_age int +-- study_start date / study_end date / gender_concept_ids varchar(100) +-- time_to_event_settings bit / dechallenge_rechallenge_settings bit +-- target_baseline_settings bit / risk_factor_settings bit / case_series_settings bit + +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN nesting_cohort_id BIGINT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN min_age INT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN max_age INT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN study_start DATE; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN study_end DATE; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN gender_concept_ids VARCHAR(100); +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN time_to_event_settings BIT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN dechallenge_rechallenge_settings BIT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN target_baseline_settings BIT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN risk_factor_settings BIT; +ALTER TABLE @database_schema.@table_prefixtarget_settings +ADD COLUMN case_series_settings BIT; + + +-- =========================== +-- 7) Add/remove columns in case_settings +-- =========================== +-- case_settings: +-- remove: runtype +ALTER TABLE @database_schema.@table_prefixcase_settings DROP COLUMN runtype; +-- add: risk_factor_settings varchar(50) / case_series_settings varchar(50) +ALTER TABLE @database_schema.@table_prefixcase_settings +ADD COLUMN risk_factor_settings VARCHAR(50); +ALTER TABLE @database_schema.@table_prefixcase_settings +ADD COLUMN case_series_settings VARCHAR(50); + diff --git a/tests/testthat/test-manualData.R b/tests/testthat/test-manualData.R index 4d3245b..5d90937 100644 --- a/tests/testthat/test-manualData.R +++ b/tests/testthat/test-manualData.R @@ -1,12 +1,14 @@ context("manual data") manualData <- file.path(tempdir(), "manual.sqlite") -on.exit(file.remove(manualData), add = TRUE) +on.exit(unlink(manualData, force = TRUE), add = TRUE) manualData2 <- file.path(tempdir(), "manual2.sqlite") -on.exit(file.remove(manualData2), add = TRUE) +on.exit(unlink(manualData2, force = TRUE), add = TRUE) test_that("manual data runCharacterizationAnalyses", { + skipIfCreateTargetCohortSqlUnavailable() + # this test creates made-up OMOP CDM data # and runs runCharacterizationAnalyses on the data # to check whether the results are as expected @@ -147,19 +149,23 @@ test_that("manual data runCharacterizationAnalyses", { ) # create settings and run + manualStudyPopulation <- createStudyPopulationSettings( + targetIds = 1, + limitToFirstInNDays = 99999, + minPriorObservation = 365 + ) + characterizationSettings <- createCharacterizationSettings( timeToEventSettings = createTimeToEventSettings( - targetIds = 1, + studyPopulationSettings = manualStudyPopulation, outcomeIds = 2 ), dechallengeRechallengeSettings = createDechallengeRechallengeSettings( - targetIds = 1, + studyPopulationSettings = manualStudyPopulation, outcomeIds = 2 ), targetBaselineSettings = createTargetBaselineSettings( - targetIds = 1, - limitToFirstInNDays = 99999, - minPriorObservation = 365, + studyPopulationSettings = manualStudyPopulation, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsAge = TRUE, useDemographicsGender = TRUE, @@ -168,10 +174,8 @@ test_that("manual data runCharacterizationAnalyses", { ), riskFactorSettings = createRiskFactorSettings( - targetIds = 1, + studyPopulationSettings = manualStudyPopulation, outcomeIds = 2, - limitToFirstInNDays = 99999, - minPriorObservation = 365, outcomeWashoutDays = 30, riskWindowStart = 1, riskWindowEnd = 90, @@ -183,10 +187,8 @@ test_that("manual data runCharacterizationAnalyses", { ), caseSeriesSettings = createCaseSeriesSettings( - targetIds = 1, + studyPopulationSettings = manualStudyPopulation, outcomeIds = 2, - limitToFirstInNDays = 99999, - minPriorObservation = 365, outcomeWashoutDays = 30, riskWindowStart = 1, riskWindowEnd = 90, @@ -201,6 +203,8 @@ test_that("manual data runCharacterizationAnalyses", { targetTable = "cohort", outcomeDatabaseSchema = schema, outcomeTable = "cohort", + nestingCohortTable = "cohort", + nestingCohortDatabaseSchema = schema, cdmDatabaseSchema = schema, characterizationSettings = characterizationSettings, outputDirectory = file.path(tempdir(), "result"), @@ -273,7 +277,7 @@ test_that("manual data runCharacterizationAnalyses", { testthat::expect_true(tset$min_prior_observation == 365) testthat::expect_true(tset$characterization_target_id == tset$target_id*10) - attrition <- utils::read.csv(file.path(tempdir(), "result", "c_attrition.csv")) + attrition <- utils::read.csv(file.path(tempdir(), "result", "c_target_attrition.csv")) # there should be 9 people as the first subject has cohort date outside observation testthat::expect_true(attrition$n[attrition$cohort_definition_id==10] == 9) diff --git a/tests/testthat/test-viewShiny.R b/tests/testthat/test-viewShiny.R index 006ffbb..885265b 100644 --- a/tests/testthat/test-viewShiny.R +++ b/tests/testthat/test-viewShiny.R @@ -16,34 +16,40 @@ test_that("ensure_installed", { }) test_that("prepareCharacterizationShiny works", { + skipIfCreateTargetCohortSqlUnavailable() + targetIds <- c(1, 2, 4) outcomeIds <- c(3) + studyPop1 <- createStudyPopulationSettings(targetIds = 1) + studyPop2 <- createStudyPopulationSettings(targetIds = 2) + studyPopAll <- createStudyPopulationSettings(targetIds = targetIds) + timeToEventSettings1 <- createTimeToEventSettings( - targetIds = 1, + studyPopulationSettings = studyPop1, outcomeIds = c(3, 4) ) timeToEventSettings2 <- createTimeToEventSettings( - targetIds = 2, + studyPopulationSettings = studyPop2, outcomeIds = c(3, 4) ) dechallengeRechallengeSettings <- createDechallengeRechallengeSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, outcomeIds = outcomeIds, dechallengeStopInterval = 30, dechallengeEvaluationWindow = 31 ) targetSettings1 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsGender = TRUE ) ) targetSettings2 <- createTargetBaselineSettings( - targetIds = targetIds, + studyPopulationSettings = studyPopAll, covariateSettings = FeatureExtraction::createCovariateSettings( useDemographicsAge = TRUE, useDemographicsRace = TRUE @@ -71,6 +77,8 @@ test_that("prepareCharacterizationShiny works", { targetTable = "cohort", outcomeDatabaseSchema = "main", outcomeTable = "cohort", + nestingCohortDatabaseSchema = 'main', + nestingCohortTable = "cohort", outputDatabaseSchema = 'main', outputTable = 'char_cohort', From 20f11462ede857df12166e10806dc04457ea794a Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 13:29:45 -0400 Subject: [PATCH 06/39] fixing tests and example -replacing IFNULL with ISNULL - fixing outcomeId is dechal-rechal - fixing manual data tests --- R/Database.R | 6 ++- R/DechallengeRechallenge.R | 4 +- R/RunCharacterization.R | 6 ++- .../sql_server/RechallengeFailCaseSeries.sql | 52 ++++++++++++------- .../sql_server/RiskFactorBinaryExtraction.sql | 28 +++++----- .../RiskFactorContinuousExtraction.sql | 50 +++++++++--------- man/insertResultsToDatabase.Rd | 6 ++- man/runCharacterizationAnalyses.Rd | 6 ++- tests/testthat/test-manualData.R | 29 +++++++---- 9 files changed, 113 insertions(+), 74 deletions(-) diff --git a/R/Database.R b/R/Database.R index 000e1f2..226cd92 100644 --- a/R/Database.R +++ b/R/Database.R @@ -81,7 +81,11 @@ createSqliteDatabase <- function( #' #conDet <- exampleOmopConnectionDetails() #' #' #tteSet <- createTimeToEventSettings( -#' #targetIds = c(1,2), +#' # studyPopulationSettings = createStudyPopulationSettings( +#' # targetIds = c(1,2), +#' # limitToFirstInNDays = 0, +#' # minPriorObservation = 0 +#' # ), #' # outcomeIds = 3 #' # ) #' diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index 7be864a..2beb2fe 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -167,7 +167,7 @@ computeDechallengeRechallengeAnalyses <- function( outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow ) @@ -308,7 +308,7 @@ computeRechallengeFailCaseSeriesAnalyses <- function( outcome_database_schema = outcomeDatabaseSchema, outcome_table = outcomeTable, characterization_target_ids = paste(settings$characterizationTargetIds, sep = "", collapse = ","), - outcome_ids = paste(settings$outcomeCohortDefinitionIds, sep = "", collapse = ","), + outcome_ids = paste(settings$outcomeIds, sep = "", collapse = ","), dechallenge_stop_interval = settings$dechallengeStopInterval, dechallenge_evaluation_window = settings$dechallengeEvaluationWindow, show_subject_id = showSubjectId diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 42126e6..fcaaebb 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -312,7 +312,11 @@ loadCharacterizationSettings <- function( #' conDet <- exampleOmopConnectionDetails() #' #' tteSet <- createTimeToEventSettings( -#' targetIds = c(1,2), +#' studyPopulationSettings = createStudyPopulationSettings( +#' targetIds = c(1,2), +#' limitToFirstInNDays = 0, +#' minPriorObservation = 0 +#' ), #' outcomeIds = 3 #' ) #' diff --git a/inst/sql/sql_server/RechallengeFailCaseSeries.sql b/inst/sql/sql_server/RechallengeFailCaseSeries.sql index 74dcfd1..75ac3bc 100644 --- a/inst/sql/sql_server/RechallengeFailCaseSeries.sql +++ b/inst/sql/sql_server/RechallengeFailCaseSeries.sql @@ -20,16 +20,8 @@ ON sc.subject_id = tc.subject_id AND sc.cohort_start_date = tc.cohort_start_date AND sc.cohort_definition_id = ts.characterization_target_id WHERE sc.cohort_definition_id in (@characterization_target_ids) ---where cohort_definition_id in (@target_ids) ; - ---IF OBJECT_ID('tempdb..#target_cohort', 'U') IS NOT NULL DROP TABLE #target_cohort; ---select * into #target_cohort ---from @target_database_schema.@target_table ---where cohort_definition_id in (@target_ids) ---; - IF OBJECT_ID('tempdb..#outcome_cohort', 'U') IS NOT NULL DROP TABLE #outcome_cohort; select * into #outcome_cohort from @outcome_database_schema.@outcome_table @@ -62,32 +54,52 @@ select into #fail_case_series from - (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) dc0 + (select *, + cohort_start_date as dc0_start_date, + cohort_end_date as dc0_end_date, + row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort + ) dc0 inner join - (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) dc1 + (select *, + cohort_start_date as dc1_start_date, + cohort_end_date as dc1_end_date, + row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort + ) dc1 on dc0.subject_id = dc1.subject_id and dc0.cohort_definition_id = dc1.cohort_definition_id and dc0.era_number = 1 inner join - (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) io1 + (select *, + cohort_start_date as io1_start_date, + cohort_end_date as io1_end_date, + row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort + ) io1 on dc1.subject_id = io1.subject_id - and io1.cohort_start_date > dc1.cohort_start_date and io1.cohort_start_date <= dc1.cohort_end_date - and dc1.cohort_end_date <= dateadd(day,@dechallenge_stop_interval,io1.cohort_start_date) -- exposure ends shortly after outcome starts + and io1.io1_start_date > dc1.dc1_start_date and io1.io1_start_date <= dc1.dc1_end_date + and dc1.dc1_end_date <= dateadd(day,@dechallenge_stop_interval,io1.io1_start_date) -- exposure ends shortly after outcome starts left join #outcome_cohort ro0 -- used to exclude people who have the outcome between exposure or next eligible time on dc1.subject_id = ro0.subject_id and io1.cohort_definition_id = ro0.cohort_definition_id - and ro0.cohort_start_date > dc1.cohort_end_date - and ro0.cohort_start_date <= dateadd(day,@dechallenge_evaluation_window,dc1.cohort_end_date) --this should be parameterized to be the dechallenge window required for success/failure + and ro0.cohort_start_date > dc1.dc1_end_date + and ro0.cohort_start_date <= dateadd(day,@dechallenge_evaluation_window,dc1.dc1_end_date) --this should be parameterized to be the dechallenge window required for success/failure inner join - (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort) de1 + (select *, + cohort_start_date as de1_start_date, + cohort_end_date as de1_end_date, + row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #target_cohort + ) de1 on dc1.subject_id = de1.subject_id and dc1.cohort_definition_id = de1.cohort_definition_id - and de1.cohort_start_date > dateadd(day,@dechallenge_evaluation_window,dc1.cohort_end_date) --using same dechallenge window to detrmine when rechallenge attempt can start + and de1.de1_start_date > dateadd(day,@dechallenge_evaluation_window,dc1.dc1_end_date) --using same dechallenge window to detrmine when rechallenge attempt can start inner join - (select *, row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort) ro1 + (select *, + cohort_start_date as ro1_start_date, + cohort_end_date as ro1_end_date, + row_number() over (partition by cohort_definition_id, subject_id order by cohort_start_date) as era_number from #outcome_cohort + ) ro1 on de1.subject_id = ro1.subject_id and io1.cohort_definition_id = ro1.cohort_definition_id - and ro1.cohort_start_date > de1.cohort_start_date - and ro1.cohort_start_date <= de1.cohort_end_date + and ro1.ro1_start_date > de1.de1_start_date + and ro1.ro1_start_date <= de1.de1_end_date where ro0.subject_id is null ; diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index 44dac83..29837cd 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -31,25 +31,25 @@ SELECT * FROM ( SELECT -IFNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, -IFNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, -IFNULL(non_case_sum_value, 0) as non_case_sum_value, -IFNULL(case_sum_value, 0) as case_sum_value, -IFNULL(non_case_average_value, 0) as non_case_average_value, -IFNULL(case_average_value, 0) as case_average_value, -(IFNULL(case_average_value, 0.0) - IFNULL(non_case_average_value, 0.0))/ +ISNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, +ISNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, +ISNULL(non_case_sum_value, 0) as non_case_sum_value, +ISNULL(case_sum_value, 0) as case_sum_value, +ISNULL(non_case_average_value, 0) as non_case_average_value, +ISNULL(case_average_value, 0) as case_average_value, +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ SQRT( ( ( - (POWER((1.0 - IFNULL(case_average_value, 0.0)),2) * IFNULL(case_sum_value*1.0, 0.0)) + - (POWER((0.0 - IFNULL(case_average_value, 0.0)),2) * (IFNULL(case_n*1.0, 0.0) - IFNULL(case_sum_value*1.0, 0.0))) - )/CASE WHEN IFNULL(case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE IFNULL(case_n*1.0-1.0, 1.0) END + (POWER((1.0 - ISNULL(case_average_value, 0.0)),2) * ISNULL(case_sum_value*1.0, 0.0)) + + (POWER((0.0 - ISNULL(case_average_value, 0.0)),2) * (ISNULL(case_n*1.0, 0.0) - ISNULL(case_sum_value*1.0, 0.0))) + )/CASE WHEN ISNULL(case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(case_n*1.0-1.0, 1.0) END + ( - (POWER((1.0 - IFNULL(non_case_average_value, 0.0)),2) * IFNULL(non_case_sum_value*1.0, 0.0)) + - (POWER((0.0 - IFNULL(non_case_average_value, 0.0)),2) * (IFNULL(non_case_n*1.0, 0.0) - IFNULL(non_case_sum_value*1.0, 0))) - )/CASE WHEN IFNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE IFNULL(non_case_n*1.0-1.0, 1.0) END + (POWER((1.0 - ISNULL(non_case_average_value, 0.0)),2) * ISNULL(non_case_sum_value*1.0, 0.0)) + + (POWER((0.0 - ISNULL(non_case_average_value, 0.0)),2) * (ISNULL(non_case_n*1.0, 0.0) - ISNULL(non_case_sum_value*1.0, 0))) + )/CASE WHEN ISNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(non_case_n*1.0-1.0, 1.0) END )/2.0 ) as standardized_mean_difference @@ -90,6 +90,6 @@ AND non_cases.covariate_id = cases.covariate_id ) smd_table WHERE abs(smd_table.standardized_mean_difference) >= @smd_min -AND (IFNULL(non_case_sum_value, 0) + IFNULL(case_sum_value, 0) ) >= @min_count +AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count ; diff --git a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql index 6b453f1..dd1dcf2 100644 --- a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql +++ b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql @@ -34,31 +34,31 @@ SELECT * FROM ( SELECT -IFNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, -IFNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, -IFNULL(non_case_count_value, 0) as non_case_count_value, -IFNULL(case_count_value, 0) as case_count_value, -IFNULL(non_case_min_value, 0) as non_case_min_value, -IFNULL(case_min_value, 0) as case_min_value, -IFNULL(non_case_max_value, 0) as non_case_max_value, -IFNULL(case_max_value, 0) as case_max_value, -IFNULL(non_case_average_value, 0) as non_case_average_value, -IFNULL(case_average_value, 0) as case_average_value, -IFNULL(non_case_median_value, 0) as non_case_median_value, -IFNULL(case_median_value, 0) as case_median_value, -IFNULL(non_case_p10_value, 0) as non_case_p10_value, -IFNULL(case_p10_value, 0) as case_p10_value, -IFNULL(non_case_p25_value, 0) as non_case_p25_value, -IFNULL(case_p25_value, 0) as case_p25_value, -IFNULL(non_case_p75_value, 0) as non_case_p75_value, -IFNULL(case_p75_value, 0) as case_p75_value, -IFNULL(non_case_p90_value, 0) as non_case_p90_value, -IFNULL(case_p90_value, 0) as case_p90_value, -IFNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, -IFNULL(case_standard_deviation, 0) as case_standard_deviation, -(IFNULL(case_average_value, 0.0) - IFNULL(non_case_average_value, 0.0))/ +ISNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, +ISNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, +ISNULL(non_case_count_value, 0) as non_case_count_value, +ISNULL(case_count_value, 0) as case_count_value, +ISNULL(non_case_min_value, 0) as non_case_min_value, +ISNULL(case_min_value, 0) as case_min_value, +ISNULL(non_case_max_value, 0) as non_case_max_value, +ISNULL(case_max_value, 0) as case_max_value, +ISNULL(non_case_average_value, 0) as non_case_average_value, +ISNULL(case_average_value, 0) as case_average_value, +ISNULL(non_case_median_value, 0) as non_case_median_value, +ISNULL(case_median_value, 0) as case_median_value, +ISNULL(non_case_p10_value, 0) as non_case_p10_value, +ISNULL(case_p10_value, 0) as case_p10_value, +ISNULL(non_case_p25_value, 0) as non_case_p25_value, +ISNULL(case_p25_value, 0) as case_p25_value, +ISNULL(non_case_p75_value, 0) as non_case_p75_value, +ISNULL(case_p75_value, 0) as case_p75_value, +ISNULL(non_case_p90_value, 0) as non_case_p90_value, +ISNULL(case_p90_value, 0) as case_p90_value, +ISNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, +ISNULL(case_standard_deviation, 0) as case_standard_deviation, +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ SQRT( -(POWER(IFNULL(case_standard_deviation, 0.0),2) + POWER(IFNULL(non_case_standard_deviation, 0.0),2)) +(POWER(ISNULL(case_standard_deviation, 0.0),2) + POWER(ISNULL(non_case_standard_deviation, 0.0),2)) /2.0) as standardized_mean_difference @@ -113,5 +113,5 @@ AND non_cases.covariate_id = cases.covariate_id ) temp WHERE abs(temp.standardized_mean_difference) >= @smd_min -AND (IFNULL(non_case_count_value, 0) + IFNULL(case_count_value, 0) ) >= @min_count +AND (ISNULL(non_case_count_value, 0) + ISNULL(case_count_value, 0) ) >= @min_count ; diff --git a/man/insertResultsToDatabase.Rd b/man/insertResultsToDatabase.Rd index 9e5f136..37ba802 100644 --- a/man/insertResultsToDatabase.Rd +++ b/man/insertResultsToDatabase.Rd @@ -41,7 +41,11 @@ Calls ResultModelManager uploadResults function to upload the csv files #conDet <- exampleOmopConnectionDetails() #tteSet <- createTimeToEventSettings( -#targetIds = c(1,2), +# studyPopulationSettings = createStudyPopulationSettings( +# targetIds = c(1,2), +# limitToFirstInNDays = 0, +# minPriorObservation = 0 +# ), # outcomeIds = 3 # ) diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index 2a580ce..3b7966f 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -114,7 +114,11 @@ specified saveDirectory conDet <- exampleOmopConnectionDetails() tteSet <- createTimeToEventSettings( - targetIds = c(1,2), + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = 3 ) diff --git a/tests/testthat/test-manualData.R b/tests/testthat/test-manualData.R index 5d90937..508f5cd 100644 --- a/tests/testthat/test-manualData.R +++ b/tests/testthat/test-manualData.R @@ -155,13 +155,19 @@ test_that("manual data runCharacterizationAnalyses", { minPriorObservation = 365 ) + manualStudyPopulationDc <- createStudyPopulationSettings( + targetIds = 1, + limitToFirstInNDays = 0, + minPriorObservation = 0 + ) + characterizationSettings <- createCharacterizationSettings( timeToEventSettings = createTimeToEventSettings( studyPopulationSettings = manualStudyPopulation, outcomeIds = 2 ), dechallengeRechallengeSettings = createDechallengeRechallengeSettings( - studyPopulationSettings = manualStudyPopulation, + studyPopulationSettings = manualStudyPopulationDc, outcomeIds = 2 ), targetBaselineSettings = createTargetBaselineSettings( @@ -197,6 +203,7 @@ test_that("manual data runCharacterizationAnalyses", { ) ) ) + runCharacterizationAnalyses( connectionDetails = connectionDetailsManual, targetDatabaseSchema = schema, @@ -243,8 +250,12 @@ test_that("manual data runCharacterizationAnalyses", { # TODO: check in code whether minCellCount < or <= dechal <- utils::read.csv(file.path(tempdir(), "result", "c_dechallenge_rechallenge.csv")) - testthat::expect_true(dechal$num_exposure_eras == 13) - testthat::expect_true(dechal$num_persons_exposed == 10) + + # person 1 not included since target exposure is outside obs + # so 13 exposures less 1 = 12 eras and 10 people less 1 is 9 people + testthat::expect_true(dechal$num_exposure_eras == 12) + testthat::expect_true(dechal$num_persons_exposed == 9) + testthat::expect_true(dechal$num_cases == 6) testthat::expect_true(dechal$dechallenge_attempt == 5) testthat::expect_true(dechal$dechallenge_success == 5) @@ -272,14 +283,14 @@ test_that("manual data runCharacterizationAnalyses", { # targetId = 1, limitToFirstInNDays = 99999,minPriorObservation = 365, tset <- utils::read.csv(file.path(tempdir(), "result", "c_target_settings.csv")) - testthat::expect_true(tset$target_id == 1) - testthat::expect_true(tset$limit_to_first_in_n_days == 99999) - testthat::expect_true(tset$min_prior_observation == 365) - testthat::expect_true(tset$characterization_target_id == tset$target_id*10) + testthat::expect_true(tset$target_id[2] == 1) + testthat::expect_true(tset$limit_to_first_in_n_days[2] == 99999) + testthat::expect_true(tset$min_prior_observation[2] == 365) + testthat::expect_true(tset$characterization_target_id[2] == 2*10) attrition <- utils::read.csv(file.path(tempdir(), "result", "c_target_attrition.csv")) # there should be 9 people as the first subject has cohort date outside observation - testthat::expect_true(attrition$n[attrition$cohort_definition_id==10] == 9) + testthat::expect_true(max(attrition$n_people[attrition$characterization_target_id==10]) == 9) # useDemographicsAge = TRUE, useDemographicsGender = TRUE, useConditionEraAnyTimePrior = TRUE covs <- utils::read.csv(file.path(tempdir(), "result", "c_target_covariates.csv")) @@ -291,7 +302,7 @@ test_that("manual data runCharacterizationAnalyses", { # data is all female so make sure female cov has average_value of 1 testthat::expect_true(covs$average_value[covs$covariate_id == 8532001] == 1) - testthat::expect_true(covs$sum_value[covs$covariate_id == 8532001] == attrition$n[attrition$cohort_definition_id==10]) + testthat::expect_true(covs$sum_value[covs$covariate_id == 8532001] == max(attrition$n_people[attrition$characterization_target_id==10])) covs_cont <- utils::read.csv(file.path(tempdir(), "result", "c_target_covariates_continuous.csv")) testthat::expect_true(1002 %in% covs_cont$covariate_id) From dfa8a2d5fb19d9ea1b47d74a1eda552c34219c57 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 13:32:30 -0400 Subject: [PATCH 07/39] replacing IFNULL with ISNULL - replacing IFNULL with ISNULL --- .../sql_server/RiskFactorBinaryExtraction.sql | 28 +++++------ .../RiskFactorContinuousExtraction.sql | 50 +++++++++---------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index 44dac83..29837cd 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -31,25 +31,25 @@ SELECT * FROM ( SELECT -IFNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, -IFNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, -IFNULL(non_case_sum_value, 0) as non_case_sum_value, -IFNULL(case_sum_value, 0) as case_sum_value, -IFNULL(non_case_average_value, 0) as non_case_average_value, -IFNULL(case_average_value, 0) as case_average_value, -(IFNULL(case_average_value, 0.0) - IFNULL(non_case_average_value, 0.0))/ +ISNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, +ISNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, +ISNULL(non_case_sum_value, 0) as non_case_sum_value, +ISNULL(case_sum_value, 0) as case_sum_value, +ISNULL(non_case_average_value, 0) as non_case_average_value, +ISNULL(case_average_value, 0) as case_average_value, +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ SQRT( ( ( - (POWER((1.0 - IFNULL(case_average_value, 0.0)),2) * IFNULL(case_sum_value*1.0, 0.0)) + - (POWER((0.0 - IFNULL(case_average_value, 0.0)),2) * (IFNULL(case_n*1.0, 0.0) - IFNULL(case_sum_value*1.0, 0.0))) - )/CASE WHEN IFNULL(case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE IFNULL(case_n*1.0-1.0, 1.0) END + (POWER((1.0 - ISNULL(case_average_value, 0.0)),2) * ISNULL(case_sum_value*1.0, 0.0)) + + (POWER((0.0 - ISNULL(case_average_value, 0.0)),2) * (ISNULL(case_n*1.0, 0.0) - ISNULL(case_sum_value*1.0, 0.0))) + )/CASE WHEN ISNULL(case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(case_n*1.0-1.0, 1.0) END + ( - (POWER((1.0 - IFNULL(non_case_average_value, 0.0)),2) * IFNULL(non_case_sum_value*1.0, 0.0)) + - (POWER((0.0 - IFNULL(non_case_average_value, 0.0)),2) * (IFNULL(non_case_n*1.0, 0.0) - IFNULL(non_case_sum_value*1.0, 0))) - )/CASE WHEN IFNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE IFNULL(non_case_n*1.0-1.0, 1.0) END + (POWER((1.0 - ISNULL(non_case_average_value, 0.0)),2) * ISNULL(non_case_sum_value*1.0, 0.0)) + + (POWER((0.0 - ISNULL(non_case_average_value, 0.0)),2) * (ISNULL(non_case_n*1.0, 0.0) - ISNULL(non_case_sum_value*1.0, 0))) + )/CASE WHEN ISNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(non_case_n*1.0-1.0, 1.0) END )/2.0 ) as standardized_mean_difference @@ -90,6 +90,6 @@ AND non_cases.covariate_id = cases.covariate_id ) smd_table WHERE abs(smd_table.standardized_mean_difference) >= @smd_min -AND (IFNULL(non_case_sum_value, 0) + IFNULL(case_sum_value, 0) ) >= @min_count +AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count ; diff --git a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql index 6b453f1..dd1dcf2 100644 --- a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql +++ b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql @@ -34,31 +34,31 @@ SELECT * FROM ( SELECT -IFNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, -IFNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, -IFNULL(non_case_count_value, 0) as non_case_count_value, -IFNULL(case_count_value, 0) as case_count_value, -IFNULL(non_case_min_value, 0) as non_case_min_value, -IFNULL(case_min_value, 0) as case_min_value, -IFNULL(non_case_max_value, 0) as non_case_max_value, -IFNULL(case_max_value, 0) as case_max_value, -IFNULL(non_case_average_value, 0) as non_case_average_value, -IFNULL(case_average_value, 0) as case_average_value, -IFNULL(non_case_median_value, 0) as non_case_median_value, -IFNULL(case_median_value, 0) as case_median_value, -IFNULL(non_case_p10_value, 0) as non_case_p10_value, -IFNULL(case_p10_value, 0) as case_p10_value, -IFNULL(non_case_p25_value, 0) as non_case_p25_value, -IFNULL(case_p25_value, 0) as case_p25_value, -IFNULL(non_case_p75_value, 0) as non_case_p75_value, -IFNULL(case_p75_value, 0) as case_p75_value, -IFNULL(non_case_p90_value, 0) as non_case_p90_value, -IFNULL(case_p90_value, 0) as case_p90_value, -IFNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, -IFNULL(case_standard_deviation, 0) as case_standard_deviation, -(IFNULL(case_average_value, 0.0) - IFNULL(non_case_average_value, 0.0))/ +ISNULL(non_cases.characterization_case_id, cases.characterization_case_id) as characterization_case_id, +ISNULL(non_cases.covariate_id, cases.covariate_id) as covariate_id, +ISNULL(non_case_count_value, 0) as non_case_count_value, +ISNULL(case_count_value, 0) as case_count_value, +ISNULL(non_case_min_value, 0) as non_case_min_value, +ISNULL(case_min_value, 0) as case_min_value, +ISNULL(non_case_max_value, 0) as non_case_max_value, +ISNULL(case_max_value, 0) as case_max_value, +ISNULL(non_case_average_value, 0) as non_case_average_value, +ISNULL(case_average_value, 0) as case_average_value, +ISNULL(non_case_median_value, 0) as non_case_median_value, +ISNULL(case_median_value, 0) as case_median_value, +ISNULL(non_case_p10_value, 0) as non_case_p10_value, +ISNULL(case_p10_value, 0) as case_p10_value, +ISNULL(non_case_p25_value, 0) as non_case_p25_value, +ISNULL(case_p25_value, 0) as case_p25_value, +ISNULL(non_case_p75_value, 0) as non_case_p75_value, +ISNULL(case_p75_value, 0) as case_p75_value, +ISNULL(non_case_p90_value, 0) as non_case_p90_value, +ISNULL(case_p90_value, 0) as case_p90_value, +ISNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, +ISNULL(case_standard_deviation, 0) as case_standard_deviation, +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ SQRT( -(POWER(IFNULL(case_standard_deviation, 0.0),2) + POWER(IFNULL(non_case_standard_deviation, 0.0),2)) +(POWER(ISNULL(case_standard_deviation, 0.0),2) + POWER(ISNULL(non_case_standard_deviation, 0.0),2)) /2.0) as standardized_mean_difference @@ -113,5 +113,5 @@ AND non_cases.covariate_id = cases.covariate_id ) temp WHERE abs(temp.standardized_mean_difference) >= @smd_min -AND (IFNULL(non_case_count_value, 0) + IFNULL(case_count_value, 0) ) >= @min_count +AND (ISNULL(non_case_count_value, 0) + ISNULL(case_count_value, 0) ) >= @min_count ; From c73c9ea343f3991f4adfa3db2325a670486d31a7 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 13:33:47 -0400 Subject: [PATCH 08/39] update description --- DESCRIPTION | 4 ++-- NEWS.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7e55026..e1ccf13 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 3.0.1 -Date: 2026-4-15 +Version: 3.0.2 +Date: 2026-6-15 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), person("Patrick", "Ryan", , "ryan@ohdsi.org", role = c("aut")), diff --git a/NEWS.md b/NEWS.md index 9ff17ea..1b70e13 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +Characterization 3.0.2 +====================== +- Replacing IFNULL with ISNULL as SQL server errors with IFNULL + Characterization 3.0.1 ====================== - Fix issue with uploading results into database for shiny viewer (spacing was added to csv and causing issues and continuous covariates that are floats were incorrectly bigints) From 6015b7f3ab81843476ca2d5599ba0a125e8f349f Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 13:58:02 -0400 Subject: [PATCH 09/39] improving dechal id name consistency --- DESCRIPTION | 4 ++-- R/DechallengeRechallenge.R | 4 ++-- tests/testthat/test-dechallengeRechallenge.R | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 170bb88..2d33753 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 3.1.0 -Date: 2026-4-15 +Version: 4.0.0.9999 +Date: 2026-6-15 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), person("Patrick", "Ryan", , "ryan@ohdsi.org", role = c("aut")), diff --git a/R/DechallengeRechallenge.R b/R/DechallengeRechallenge.R index 2beb2fe..2832d7e 100644 --- a/R/DechallengeRechallenge.R +++ b/R/DechallengeRechallenge.R @@ -84,7 +84,7 @@ createDechallengeRechallengeSettings <- function( # create data.frame with all combinations result <- list( studyPopulationSettings = combineStudyPopulationSettings(studyPopulationSettings), - outcomeCohortDefinitionIds = outcomeIds, + outcomeIds = outcomeIds, dechallengeStopInterval = dechallengeStopInterval, dechallengeEvaluationWindow = dechallengeEvaluationWindow ) @@ -382,7 +382,7 @@ getDechallengeRechallengeJobs <- function( characterizationSettings[[i]]$characterizationTargetIds }) outcomeIds <- lapply(ind, function(i) { - characterizationSettings[[i]]$outcomeCohortDefinitionIds + characterizationSettings[[i]]$outcomeIds }) dechallengeStopIntervals <- lapply(ind, function(i) { characterizationSettings[[i]]$dechallengeStopInterval diff --git a/tests/testthat/test-dechallengeRechallenge.R b/tests/testthat/test-dechallengeRechallenge.R index da57330..5110bec 100644 --- a/tests/testthat/test-dechallengeRechallenge.R +++ b/tests/testthat/test-dechallengeRechallenge.R @@ -35,7 +35,7 @@ test_that("createDechallengeRechallengeSettings", { ) testthat::expect_equal( - res$outcomeCohortDefinitionIds, + res$outcomeIds, outcomeIds ) From d774f81c91d50ea72a271dfa6fc8e8bd0837d23d Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 16:30:22 -0400 Subject: [PATCH 10/39] Update TargetCohorts.sql --- inst/sql/sql_server/TargetCohorts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 5b87164..8875f36 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -110,7 +110,7 @@ AND YEAR(t.cohort_start_date) - p.year_of_birth <= @max_age; {@gender_concept_ids != ''}?{ SELECT * INTO #temp_target_gender -FROM #temp_target_age +FROM #temp_target_age t INNER JOIN @cdm_database_schema.person p ON p.person_id = t.subject_id From b12ed69c92b6e074fefb5b285853b98ad55230ee Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 16:52:11 -0400 Subject: [PATCH 11/39] fixing division by 0 sqrt --- inst/sql/sql_server/RiskFactorBinaryExtraction.sql | 10 ++++++---- .../sql/sql_server/RiskFactorContinuousExtraction.sql | 11 +++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index 29837cd..fb0827e 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -27,7 +27,9 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT * +SELECT *, +CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference + FROM ( SELECT @@ -37,7 +39,7 @@ ISNULL(non_case_sum_value, 0) as non_case_sum_value, ISNULL(case_sum_value, 0) as case_sum_value, ISNULL(non_case_average_value, 0) as non_case_average_value, ISNULL(case_average_value, 0) as case_average_value, -(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))*1.0 as mean_diff, SQRT( ( ( @@ -52,7 +54,7 @@ SQRT( )/CASE WHEN ISNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(non_case_n*1.0-1.0, 1.0) END )/2.0 - ) as standardized_mean_difference + ) as st_dev FROM @@ -89,7 +91,7 @@ AND non_cases.covariate_id = cases.covariate_id ) smd_table -WHERE abs(smd_table.standardized_mean_difference) >= @smd_min +WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff/0.0000001 ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count ; diff --git a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql index dd1dcf2..2829029 100644 --- a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql +++ b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql @@ -30,7 +30,10 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT * +SELECT *, +CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference + + FROM ( SELECT @@ -56,10 +59,10 @@ ISNULL(non_case_p90_value, 0) as non_case_p90_value, ISNULL(case_p90_value, 0) as case_p90_value, ISNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, ISNULL(case_standard_deviation, 0) as case_standard_deviation, -(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))*1.0 as mean_diff, SQRT( (POWER(ISNULL(case_standard_deviation, 0.0),2) + POWER(ISNULL(non_case_standard_deviation, 0.0),2)) -/2.0) as standardized_mean_difference +/2.0) as st_dev FROM @@ -112,6 +115,6 @@ AND non_cases.covariate_id = cases.covariate_id ) temp -WHERE abs(temp.standardized_mean_difference) >= @smd_min +WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_count_value, 0) + ISNULL(case_count_value, 0) ) >= @min_count ; From 67fe8ade6ff878ab4c379d6fdaaa12d32071a073 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 17:02:39 -0400 Subject: [PATCH 12/39] Update RiskFactorBinaryExtraction.sql --- inst/sql/sql_server/RiskFactorBinaryExtraction.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index fb0827e..fb75c32 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -91,7 +91,7 @@ AND non_cases.covariate_id = cases.covariate_id ) smd_table -WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff/0.0000001 ELSE mean_diff/st_dev END) >= @smd_min +WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count ; From 95ff2677f03effbb23091a77777c9161c4ed2120 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 17:15:19 -0400 Subject: [PATCH 13/39] fixing sqrt 0 division fixing sqrt 0 division --- .../sql_server/RiskFactorBinaryExtraction.sql | 16 ++++++--- .../RiskFactorContinuousExtraction.sql | 33 ++++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index 29837cd..62b56b4 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -27,7 +27,15 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT * +SELECT +characterization_case_id, +covariate_id, +non_case_sum_value, +case_sum_value, +non_case_average_value, +case_average_value, +CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference + FROM ( SELECT @@ -37,7 +45,7 @@ ISNULL(non_case_sum_value, 0) as non_case_sum_value, ISNULL(case_sum_value, 0) as case_sum_value, ISNULL(non_case_average_value, 0) as non_case_average_value, ISNULL(case_average_value, 0) as case_average_value, -(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))*1.0 as mean_diff, SQRT( ( ( @@ -52,7 +60,7 @@ SQRT( )/CASE WHEN ISNULL(non_case_n*1.0-1.0, 1.0) = 0 THEN 1.0 ELSE ISNULL(non_case_n*1.0-1.0, 1.0) END )/2.0 - ) as standardized_mean_difference + ) as st_dev FROM @@ -89,7 +97,7 @@ AND non_cases.covariate_id = cases.covariate_id ) smd_table -WHERE abs(smd_table.standardized_mean_difference) >= @smd_min +WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count ; diff --git a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql index dd1dcf2..bf44319 100644 --- a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql +++ b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql @@ -30,7 +30,32 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT * +SELECT +characterization_case_id, +covariate_id, +non_case_count_value, +case_count_value, +non_case_min_value, +case_min_value, +non_case_max_value, +case_max_value, +non_case_average_value, +case_average_value, +non_case_median_value, +case_median_value, +non_case_p10_value, +case_p10_value, +non_case_p25_value, +case_p25_value, +non_case_p75_value, +case_p75_value, +non_case_p90_value, +case_p90_value, +non_case_standard_deviation, +case_standard_deviation, + +CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference + FROM ( SELECT @@ -56,10 +81,10 @@ ISNULL(non_case_p90_value, 0) as non_case_p90_value, ISNULL(case_p90_value, 0) as case_p90_value, ISNULL(non_case_standard_deviation, 0) as non_case_standard_deviation, ISNULL(case_standard_deviation, 0) as case_standard_deviation, -(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))/ +(ISNULL(case_average_value, 0.0) - ISNULL(non_case_average_value, 0.0))*1.0 as mean_diff, SQRT( (POWER(ISNULL(case_standard_deviation, 0.0),2) + POWER(ISNULL(non_case_standard_deviation, 0.0),2)) -/2.0) as standardized_mean_difference +/2.0) as st_dev FROM @@ -112,6 +137,6 @@ AND non_cases.covariate_id = cases.covariate_id ) temp -WHERE abs(temp.standardized_mean_difference) >= @smd_min +WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_count_value, 0) + ISNULL(case_count_value, 0) ) >= @min_count ; From 2838eb4cacaf85378541c54d988af0862c15e493 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 16 Jun 2026 17:21:56 -0400 Subject: [PATCH 14/39] manually specifying risk factor columns manually specifying risk factor columns to remove mean_diff and st_dev --- .../sql_server/RiskFactorBinaryExtraction.sql | 8 ++++++- .../RiskFactorContinuousExtraction.sql | 24 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index fb75c32..62b56b4 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -27,7 +27,13 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT *, +SELECT +characterization_case_id, +covariate_id, +non_case_sum_value, +case_sum_value, +non_case_average_value, +case_average_value, CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference FROM diff --git a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql index 2829029..59b14d1 100644 --- a/inst/sql/sql_server/RiskFactorContinuousExtraction.sql +++ b/inst/sql/sql_server/RiskFactorContinuousExtraction.sql @@ -30,7 +30,29 @@ SELECT case_id FROM cohort_of_int GROUP BY cohort_definition_id ) -SELECT *, +SELECT +characterization_case_id, +covariate_id, +non_case_count_value, +case_count_value, +non_case_min_value, +case_min_value, +non_case_max_value, +case_max_value, +non_case_average_value, +case_average_value, +non_case_median_value, +case_median_value, +non_case_p10_value, +case_p10_value, +non_case_p25_value, +case_p25_value, +non_case_p75_value, +case_p75_value, +non_case_p90_value, +case_p90_value, +non_case_standard_deviation, +case_standard_deviation, CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END as standardized_mean_difference From e382295aeb36a5a508acbaf6e4361154650d4673 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 08:58:30 -0400 Subject: [PATCH 15/39] casting potential floats into bigints aiming to fix bigquery int float issue --- inst/sql/sql_server/CaseCohorts.sql | 22 ++++++----- inst/sql/sql_server/NonCaseCohorts.sql | 52 ++++++++++++++------------ 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/inst/sql/sql_server/CaseCohorts.sql b/inst/sql/sql_server/CaseCohorts.sql index 109bdcc..553bf54 100644 --- a/inst/sql/sql_server/CaseCohorts.sql +++ b/inst/sql/sql_server/CaseCohorts.sql @@ -72,7 +72,7 @@ INSERT INTO @characterization_schema.@characterization_table( cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, char_type ) SELECT -cohort_definition_id*10+1, +CAST(cohort_definition_id*10+1 as BIGINT), row_number, subject_id, cohort_start_date, @@ -98,7 +98,7 @@ INSERT INTO @characterization_schema.@characterization_table( cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, char_type ) SELECT -cohort_definition_id*10+3, +CAST(cohort_definition_id*10+3 as BIGINT), row_number, subject_id, DATEADD(day, -@case_series_before, cohort_start_date), @@ -110,7 +110,7 @@ FROM #characterization_cases UNION SELECT -cohort_definition_id*10+4, +CAST(cohort_definition_id*10+4 as BIGINT), row_number, subject_id, DATEADD(day, 1, cohort_start_date), @@ -121,7 +121,7 @@ FROM #characterization_cases UNION SELECT -cohort_definition_id*10+5, +CAST(cohort_definition_id*10+5 as BIGINT), row_number, subject_id, DATEADD(day, 1, outcome_start_date), @@ -137,17 +137,19 @@ WHERE cohort_type = 'Cases' AND characterization_case_id in (SELECT DISTINCT cohort_definition_id FROM #characterization_cases); -INSERT INTO @characterization_schema.@case_count_table +INSERT INTO @characterization_schema.@case_count_table( +characterization_case_id, cohort_type, n_events, n_people +) SELECT -cohort_definition_id as characterization_case_id, -'Cases' as cohort_type, -count(*) as n_events, -- new -count(distinct subject_id) as n_people -- new +CAST(cohort_definition_id as BIGINT), +'Cases', +count(*), +count(distinct subject_id) FROM #characterization_cases GROUP BY -cohort_definition_id +CAST(cohort_definition_id as BIGINT) ; -- clean up diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index f555997..24e31e1 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -173,17 +173,20 @@ WHERE temp.no_tar_because_outcome_washout = 0 AND temp.no_tar_washout_and_obs = -- next DELETE FROM @characterization_schema.@case_attrition_table WHERE characterization_case_id in -(SELECT DISTINCT (cohort_definition_id-2.0)/10.0 FROM #temp_non_cases); +(SELECT DISTINCT CAST((cohort_definition_id-2.0)/10.0 AS BIGINT) FROM #temp_non_cases); -INSERT INTO @characterization_schema.@case_attrition_table +INSERT INTO @characterization_schema.@case_attrition_table( +characterization_case_id, attr_order, attr_reason, +n_events, n_people +) SELECT -(cohort_definition_id - 2.0)/10.0 as characterization_case_id, -8 as attr_order, -- new -'Has some TAR' as attr_reason, -count(*) as n_events, -- new -count(distinct subject_id) as n_people -- new +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), +8, +'Has some TAR', +count(*), +count(distinct subject_id) FROM #temp_non_cases_with_tar GROUP BY cohort_definition_id @@ -191,24 +194,27 @@ GROUP BY cohort_definition_id -- add 0s UNION SELECT DISTINCT -(cohort_definition_id - 2.0)/10.0 as characterization_case_id, -8 as attr_order, -- new -'Has some TAR' as attr_reason, -0 as n_events, -0 as n_people +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), +8, +'Has some TAR', +0, +0 FROM #temp_non_cases WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_non_cases_with_tar) ; -INSERT INTO @characterization_schema.@case_attrition_table +INSERT INTO @characterization_schema.@case_attrition_table( +characterization_case_id, attr_order, attr_reason, +n_events, n_people +) SELECT -(cohort_definition_id - 2.0)/10.0 as characterization_case_id, -9 as attr_order, -- new -'Remains after outcome washout' as attr_reason, -count(*) as n_events, -- new -count(distinct subject_id) as n_people -- new +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), +9, +'Remains after outcome washout', +count(*), +count(distinct subject_id) FROM #temp_non_cases_pass_washout GROUP BY cohort_definition_id @@ -216,11 +222,11 @@ GROUP BY cohort_definition_id UNION SELECT DISTINCT -(cohort_definition_id - 2.0)/10.0 as characterization_case_id, -9 as attr_order, -- new -'Remains after outcome washout' as attr_reason, -0 as n_events, -0 as n_people +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), +9, +'Remains after outcome washout', +0, +0 FROM #temp_non_cases_with_tar WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_non_cases_pass_washout) From 18e5f4f8edf3c2f1a6c6cf806a2d00a7f2b1c412 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 10:16:03 -0400 Subject: [PATCH 16/39] Update NonCaseCohorts.sql --- inst/sql/sql_server/NonCaseCohorts.sql | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index 24e31e1..096f5a0 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -111,14 +111,16 @@ AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_non DELETE FROM @characterization_schema.@case_count_table WHERE cohort_type = 'non-cases' AND characterization_case_id in -(SELECT DISTINCT (cohort_definition_id-2.0)/10.0 FROM #temp_non_cases); +(SELECT DISTINCT CAST((cohort_definition_id-2.0)/10.0 as BIGINT) FROM #temp_non_cases); -INSERT INTO @characterization_schema.@case_count_table +INSERT INTO @characterization_schema.@case_count_table( +characterization_case_id, cohort_type, n_events, n_people +) SELECT -(cohort_definition_id-2.0)/10.0 as characterization_case_id, -'non-cases' as cohort_type, -count(*) as n_events, -- new -count(distinct subject_id) as n_people -- new +CAST((cohort_definition_id-2.0)/10.0 as BIGINT), +'non-cases', +count(*), +count(distinct subject_id) FROM #temp_non_cases temp From fbcbf8e4ac9e215508aa89059fe5e22a83084e92 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 16:34:51 -0400 Subject: [PATCH 17/39] bigquery issue trying to fix weird union issue only impacting bigquery --- inst/sql/sql_server/NonCaseCohorts.sql | 11 +- inst/sql/sql_server/TargetCohorts.sql | 172 ++++++++++++++----------- 2 files changed, 103 insertions(+), 80 deletions(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index 096f5a0..d4d7c55 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -183,6 +183,8 @@ characterization_case_id, attr_order, attr_reason, n_events, n_people ) +SELECT * FROM +( SELECT CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), 8, @@ -192,10 +194,11 @@ count(distinct subject_id) FROM #temp_non_cases_with_tar GROUP BY cohort_definition_id +) temp -- add 0s UNION -SELECT DISTINCT +SELECT CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), 8, 'Has some TAR', @@ -211,6 +214,9 @@ INSERT INTO @characterization_schema.@case_attrition_table( characterization_case_id, attr_order, attr_reason, n_events, n_people ) + +SELECT * FROM +( SELECT CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), 9, @@ -220,10 +226,11 @@ count(distinct subject_id) FROM #temp_non_cases_pass_washout GROUP BY cohort_definition_id +) temp UNION -SELECT DISTINCT +SELECT CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), 9, 'Remains after outcome washout', diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 8875f36..0f1651b 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -175,162 +175,178 @@ DELETE FROM @characterization_schema.@target_attrition_table WHERE characterization_target_id in (SELECT DISTINCT cohort_definition_id FROM #temp_target_date) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -1 as attr_order, -- new -'Target Start' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +1, +'Target Start', +count(*), +count(distinct subject_id) FROM #temp_target GROUP BY cohort_definition_id ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -2 as attr_order, -- new -'First in @limit_to_first_in_n_days days' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +2, +'First in @limit_to_first_in_n_days days', +count(*), +count(distinct subject_id) FROM #temp_target_first GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -2 as attr_order, -- new -'First in @limit_to_first_in_n_days days' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +2, +'First in @limit_to_first_in_n_days days', +0, +0 FROM #temp_target -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_first) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -3 as attr_order, -- new -'With @min_prior_observation prior obs' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +3, +'With @min_prior_observation prior obs', +count(*), +count(distinct subject_id) FROM #temp_target_prior GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -3 as attr_order, -- new -'With @min_prior_observation prior obs' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +3, +'With @min_prior_observation prior obs', +0, +0 FROM #temp_target_first -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_prior) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -4 as attr_order, -- new -'Nested in @nesting_cohort_id' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +4, +'Nested in @nesting_cohort_id', +count(*), +count(distinct subject_id) FROM #temp_target_nest GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -4 as attr_order, -- new -'Nested in @nesting_cohort_id' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +4, +'Nested in @nesting_cohort_id', +0, +0 FROM #temp_target_prior -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_nest) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -5 as attr_order, -- new -'Aged @min_age to @max_age' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +5, +'Aged @min_age to @max_age', +count(*), +count(distinct subject_id) FROM #temp_target_age GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -5 as attr_order, -- new -'Aged @min_age to @max_age' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +5, -- new +'Aged @min_age to @max_age', +0, +0 FROM #temp_target_nest -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_age) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people) SELECT -cohort_definition_id as characterization_target_id, -6 as attr_order, -- new -'Gender in @gender_concept_ids' as attr_reason, -count(*) as n_events, -count(distinct subject_id) as n_people +cohort_definition_id, +6, +'Gender in @gender_concept_ids', +count(*), +count(distinct subject_id) FROM #temp_target_gender GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -6 as attr_order, -- new -'Gender in @gender_concept_ids' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +6, +'Gender in @gender_concept_ids', +0, +0 FROM #temp_target_age -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_gender) ; -INSERT INTO @characterization_schema.@target_attrition_table +INSERT INTO @characterization_schema.@target_attrition_table( +characterization_target_id, attr_order, attr_reason, +n_events, n_people +) SELECT -cohort_definition_id as characterization_target_id, -7 as attr_order, -- new -'Starting between @study_start to @study_end' as attr_reason, -count(*) as n_events, -- new -count(distinct subject_id) as n_people -- new +cohort_definition_id, +7, +'Starting between @study_start to @study_end', +count(*), +count(distinct subject_id) + FROM #temp_target_date GROUP BY cohort_definition_id -- add 0s UNION -SELECT DISTINCT -cohort_definition_id as characterization_target_id, -7 as attr_order, -- new -'Starting between @study_start to @study_end' as attr_reason, -0 as n_events, -0 as n_people +SELECT +cohort_definition_id, +7, +'Starting between @study_start to @study_end', +0, +0 FROM #temp_target_gender -- preious WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_date) From 7f9d96b85f1a38d70cb633b6bea55e20e0456007 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 19:30:15 -0400 Subject: [PATCH 18/39] Update TargetCohorts.sql adding spacing between UNION and SELECT to fix odd bigquery conversion --- inst/sql/sql_server/TargetCohorts.sql | 30 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 0f1651b..541dca0 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -1,7 +1,7 @@ -- first entry in washout days and min prior obs IF OBJECT_ID('tempdb..#temp_target', 'U') IS NOT NULL DROP TABLE #temp_target; -IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_fist; +IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_first; IF OBJECT_ID('tempdb..#temp_target_prior', 'U') IS NOT NULL DROP TABLE #temp_target_prior; IF OBJECT_ID('tempdb..#temp_target_nest', 'U') IS NOT NULL DROP TABLE #temp_target_nest; IF OBJECT_ID('tempdb..#temp_target_age', 'U') IS NOT NULL DROP TABLE #temp_target_age; @@ -178,6 +178,7 @@ WHERE characterization_target_id in (SELECT DISTINCT cohort_definition_id FROM # INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 1, @@ -185,13 +186,13 @@ cohort_definition_id, count(*), count(distinct subject_id) FROM #temp_target -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id ; INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 2, @@ -202,8 +203,8 @@ FROM #temp_target_first GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, 2, @@ -219,6 +220,7 @@ WHERE cohort_definition_id NOT IN INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 3, @@ -229,8 +231,8 @@ FROM #temp_target_prior GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, 3, @@ -246,6 +248,7 @@ WHERE cohort_definition_id NOT IN INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 4, @@ -256,8 +259,8 @@ FROM #temp_target_nest GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, 4, @@ -273,6 +276,7 @@ WHERE cohort_definition_id NOT IN INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 5, @@ -283,11 +287,11 @@ FROM #temp_target_age GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, -5, -- new +5, 'Aged @min_age to @max_age', 0, 0 @@ -300,6 +304,7 @@ WHERE cohort_definition_id NOT IN INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) + SELECT cohort_definition_id, 6, @@ -310,8 +315,8 @@ FROM #temp_target_gender GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, 6, @@ -328,6 +333,7 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people ) + SELECT cohort_definition_id, 7, @@ -339,15 +345,15 @@ FROM #temp_target_date GROUP BY cohort_definition_id --- add 0s UNION + SELECT cohort_definition_id, 7, 'Starting between @study_start to @study_end', 0, 0 -FROM #temp_target_gender -- preious +FROM #temp_target_gender WHERE cohort_definition_id NOT IN (SELECT distinct cohort_definition_id FROM #temp_target_date) @@ -376,7 +382,7 @@ cohort_definition_id -- ========================= -- clean up IF OBJECT_ID('tempdb..#temp_target', 'U') IS NOT NULL DROP TABLE #temp_target; -IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_fist; +IF OBJECT_ID('tempdb..#temp_target_first', 'U') IS NOT NULL DROP TABLE #temp_target_first; IF OBJECT_ID('tempdb..#temp_target_prior', 'U') IS NOT NULL DROP TABLE #temp_target_prior; IF OBJECT_ID('tempdb..#temp_target_nest', 'U') IS NOT NULL DROP TABLE #temp_target_nest; IF OBJECT_ID('tempdb..#temp_target_age', 'U') IS NOT NULL DROP TABLE #temp_target_age; From 6151af167e9607a42b7eaf3d14d6cf08c2666030 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 19:42:06 -0400 Subject: [PATCH 19/39] Update TargetCohorts.sql --- inst/sql/sql_server/TargetCohorts.sql | 40 +++++++++++++-------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 541dca0..320b380 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -193,15 +193,15 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 2, 'First in @limit_to_first_in_n_days days', count(*), count(distinct subject_id) FROM #temp_target_first -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION @@ -221,15 +221,15 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 3, 'With @min_prior_observation prior obs', count(*), count(distinct subject_id) FROM #temp_target_prior -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION @@ -249,15 +249,15 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 4, 'Nested in @nesting_cohort_id', count(*), count(distinct subject_id) FROM #temp_target_nest -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION @@ -277,15 +277,15 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 5, 'Aged @min_age to @max_age', count(*), count(distinct subject_id) FROM #temp_target_age -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION @@ -305,15 +305,15 @@ INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 6, 'Gender in @gender_concept_ids', count(*), count(distinct subject_id) FROM #temp_target_gender -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION @@ -325,16 +325,15 @@ cohort_definition_id, 0 FROM #temp_target_age -- preious WHERE cohort_definition_id NOT IN -(SELECT distinct cohort_definition_id FROM #temp_target_gender) - -; +(SELECT distinct cohort_definition_id FROM #temp_target_gender); INSERT INTO @characterization_schema.@target_attrition_table( characterization_target_id, attr_order, attr_reason, n_events, n_people ) -SELECT +SELECT * FROM +(SELECT cohort_definition_id, 7, 'Starting between @study_start to @study_end', @@ -342,8 +341,7 @@ count(*), count(distinct subject_id) FROM #temp_target_date -GROUP BY -cohort_definition_id +GROUP BY cohort_definition_id) main UNION From 6316af4a34e68878161c491544396b267a830a58 Mon Sep 17 00:00:00 2001 From: jreps Date: Wed, 17 Jun 2026 21:05:03 -0400 Subject: [PATCH 20/39] fixing another bigquery issue --- inst/sql/sql_server/NonCaseCohorts.sql | 20 ++++----- inst/sql/sql_server/TargetCohorts.sql | 60 +++++++++++++------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index d4d7c55..5a0d21a 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -186,11 +186,11 @@ n_events, n_people SELECT * FROM ( SELECT -CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), -8, -'Has some TAR', -count(*), -count(distinct subject_id) +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT) as characterization_case_id, +8 as attr_order, +'Has some TAR' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_non_cases_with_tar GROUP BY cohort_definition_id @@ -218,11 +218,11 @@ n_events, n_people SELECT * FROM ( SELECT -CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT), -9, -'Remains after outcome washout', -count(*), -count(distinct subject_id) +CAST((cohort_definition_id - 2.0)/10.0 AS BIGINT) as characterization_case_id, +9 as attr_order, +'Remains after outcome washout' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_non_cases_pass_washout GROUP BY cohort_definition_id diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 320b380..29da51b 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -195,11 +195,11 @@ n_events, n_people) SELECT * FROM (SELECT -cohort_definition_id, -2, -'First in @limit_to_first_in_n_days days', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +2 as attr_order, +'First in @limit_to_first_in_n_days days' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_first GROUP BY cohort_definition_id) main @@ -223,11 +223,11 @@ n_events, n_people) SELECT * FROM (SELECT -cohort_definition_id, -3, -'With @min_prior_observation prior obs', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +3 as attr_order, +'With @min_prior_observation prior obs' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_prior GROUP BY cohort_definition_id) main @@ -251,11 +251,11 @@ n_events, n_people) SELECT * FROM (SELECT -cohort_definition_id, -4, -'Nested in @nesting_cohort_id', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +4 as attr_order, +'Nested in @nesting_cohort_id' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_nest GROUP BY cohort_definition_id) main @@ -279,11 +279,11 @@ n_events, n_people) SELECT * FROM (SELECT -cohort_definition_id, -5, -'Aged @min_age to @max_age', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +5 as attr_order, +'Aged @min_age to @max_age' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_age GROUP BY cohort_definition_id) main @@ -307,11 +307,11 @@ n_events, n_people) SELECT * FROM (SELECT -cohort_definition_id, -6, -'Gender in @gender_concept_ids', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +6 as attr_order, +'Gender in @gender_concept_ids' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_gender GROUP BY cohort_definition_id) main @@ -334,11 +334,11 @@ n_events, n_people SELECT * FROM (SELECT -cohort_definition_id, -7, -'Starting between @study_start to @study_end', -count(*), -count(distinct subject_id) +cohort_definition_id as characterization_target_id, +7 as attr_order, +'Starting between @study_start to @study_end' as attr_reason, +count(*) as n_events, +count(distinct subject_id) as n_people FROM #temp_target_date GROUP BY cohort_definition_id) main From 494a23e1b8b861e8e9c56fd4ee4e965326a65761 Mon Sep 17 00:00:00 2001 From: jreps Date: Mon, 22 Jun 2026 11:51:17 -0400 Subject: [PATCH 21/39] minor fixes fixing missing column from migration and no cohort edge cases --- R/CaseSeries.R | 5 +++++ R/RiskFactorAnalysis.R | 18 ++++++------------ .../settings/resultsDataModelSpecification.csv | 2 +- .../Migration_2-v3_1_0_table_change.sql | 8 ++++++++ 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/R/CaseSeries.R b/R/CaseSeries.R index 56f9757..c419a7b 100644 --- a/R/CaseSeries.R +++ b/R/CaseSeries.R @@ -217,6 +217,11 @@ computeCaseSeriesAnalyses <- function( message(paste0("Case series analysis: Finding temp Ids took ", round(completionTime, digits = 1), " ", units(completionTime))) + if(nrow(caseIds) == 0){ + message('No case cohorts of minCaseSize') + return(invisible(TRUE)) + } + ## 4) run FE with all the cohorts of interest - ideally inserting the aggregate features into a new table start <- Sys.time() message("Case series analysis: Running FeatureExtraction") diff --git a/R/RiskFactorAnalysis.R b/R/RiskFactorAnalysis.R index 0c01597..e9a2012 100644 --- a/R/RiskFactorAnalysis.R +++ b/R/RiskFactorAnalysis.R @@ -245,18 +245,12 @@ computeRiskFactorAnalyses <- function( completionTime <- Sys.time() - start message(paste0("Risk factor analysis: Finding temp Ids took ", round(completionTime, digits = 1), " ", units(completionTime))) + if(length(cohortIds) == 0){ + message('No cohorts with number of people >= minSize') + return(invisible(TRUE)) + } - - ## 2) get attrition - #start <- Sys.time() - #message("Risk factor analysis: Extracting cohort attritions") - # TODO get attrition from CohortGenerator when it is in there - - #completionTime <- Sys.time() - start - #message(paste0("Risk factor analysis: Extracting cohort attritions took ", round(completionTime, digits = 1), " ", units(completionTime))) - - - ## 3) run FE with all the cohorts of interest - ideally inserting the aggregate features into a new table + ## 2) run FE with all the cohorts of interest - ideally inserting the aggregate features into a new table start <- Sys.time() message("Risk factor analysis: Running FeatureExtraction") FeatureExtraction::getDbCovariateData( @@ -287,7 +281,7 @@ computeRiskFactorAnalyses <- function( - ## 4) for each target,exclude,cases join the tables and calculate the SMD + ## 3) for each target,exclude,cases join the tables and calculate the SMD start <- Sys.time() message("Risk factor analysis: Calculating SMD for binary") diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index b0a9b77..f507d12 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -11,7 +11,7 @@ rechallenge_fail_case_series,database_id,varchar(100),Yes,Yes,No,No,The database rechallenge_fail_case_series,dechallenge_stop_interval,int,Yes,Yes,No,No,The time period that É rechallenge_fail_case_series,dechallenge_evaluation_window,int,Yes,Yes,No,No,The time period that É rechallenge_fail_case_series,characterization_target_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort -rechallenge_fail_case_series,included,bit,No,No,No,No,Whether this was included in the study population settings eras +rechallenge_fail_case_series,included,int,No,No,No,No,Whether this was included in the study population settings eras rechallenge_fail_case_series,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort rechallenge_fail_case_series,person_key,int,Yes,Yes,No,No,The dense rank for the patient (an identifier that is not the same as the database) rechallenge_fail_case_series,subject_id,bigint,No,No,No,No,The person identifier for the failed case series (optional) diff --git a/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql b/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql index b7105ca..ed152ab 100644 --- a/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql +++ b/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql @@ -145,3 +145,11 @@ ADD COLUMN risk_factor_settings VARCHAR(50); ALTER TABLE @database_schema.@table_prefixcase_settings ADD COLUMN case_series_settings VARCHAR(50); + +-- =========================== +-- 8) Add included to rechallenge_fail_case_series +-- =========================== +ALTER TABLE @database_schema.@table_prefixrechallenge_fail_case_series +ADD COLUMN included INT; + + From 6d97ab2b339713ad97541730af7024de65e9d034 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 25 Jun 2026 16:28:44 -0400 Subject: [PATCH 22/39] minor upload bug fixes - added setting lookups - changed bit columns to char(1) due to loading issues - changed boolean values to 1/0 for uploading --- DESCRIPTION | 2 +- R/CohortGeneration.R | 18 +++--- R/Database.R | 3 +- R/RunCharacterization.R | 62 +++++++++++++++++-- .../resultsDataModelSpecification.csv | 28 ++++++--- inst/sql/sql_server/UpdateVersionNumber.sql | 2 +- ...ql => Migration_2-v4_0_0_table_change.sql} | 43 ++++++++----- 7 files changed, 116 insertions(+), 42 deletions(-) rename inst/sql/sql_server/migrations/{Migration_2-v3_1_0_table_change.sql => Migration_2-v4_0_0_table_change.sql} (82%) diff --git a/DESCRIPTION b/DESCRIPTION index 2d33753..0650416 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model -Version: 4.0.0.9999 +Version: 4.0.0 Date: 2026-6-15 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 2762b6f..09a0f15 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -277,8 +277,8 @@ getCohortJobs <- function( startAnchor = x$startAnchor, riskWindowEnd = x$riskWindowEnd, endAnchor = x$endAnchor, - riskFactorSettings = TRUE, - caseSeriesSettings = FALSE + riskFactorSettings = 1, + caseSeriesSettings = 0 ) } ) @@ -315,8 +315,8 @@ getCohortJobs <- function( startAnchor = x$startAnchor, riskWindowEnd = x$riskWindowEnd, endAnchor = x$endAnchor, - riskFactorSettings = FALSE, - caseSeriesSettings = TRUE + riskFactorSettings = 0, + caseSeriesSettings = 1 ) } ) @@ -406,8 +406,8 @@ getCohortJobs <- function( .data$riskWindowEnd, .data$endAnchor ) %>% dplyr::summarize( - riskFactorSettings = any(.data$riskFactorSettings), - caseSeriesSettings = any(.data$caseSeriesSettings) + riskFactorSettings = max(.data$riskFactorSettings), + caseSeriesSettings = max(.data$caseSeriesSettings) ) %>% dplyr::ungroup() %>% dplyr::inner_join( @@ -466,8 +466,8 @@ getCohortJobs <- function( startAnchor = unique(coi$startAnchor[ind]), riskWindowEnd = unique(coi$riskWindowEnd[ind]), endAnchor = unique(coi$endAnchor[ind]), - generateRiskFactors = any(coi$riskFactorSettings[ind]) , - generateCaseSeries = any(coi$caseSeriesSettings[ind]) + generateRiskFactors = max(coi$riskFactorSettings[ind]) , + generateCaseSeries = max(coi$caseSeriesSettings[ind]) ) )), jobId = paste("cases",i, paste0(settingVal, collapse = "_"), sep = "_") @@ -475,7 +475,7 @@ getCohortJobs <- function( if(mode != 'Efficient'){ - if(any(coi$riskFactorSettings[ind])){ + if(max(coi$riskFactorSettings[ind]) == 1){ nNonCase <- nNonCase + 1 jobs <- rbind(jobs, data.frame( functionName = 'generateNonCases', diff --git a/R/Database.R b/R/Database.R index 226cd92..9e71648 100644 --- a/R/Database.R +++ b/R/Database.R @@ -331,9 +331,10 @@ migrateDataModel <- function( migrator$executeMigrations() migrator$finalize() - ParallelLogger::logInfo("Updating version number") + ParallelLogger::logInfo(paste0("Updating version number to ", utils::packageVersion("Characterization") )) updateVersionSql <- SqlRender::loadRenderTranslateSql("UpdateVersionNumber.sql", packageName = utils::packageName(), + version_number = utils::packageVersion("Characterization"), database_schema = databaseSchema, table_prefix = tablePrefix, dbms = connectionDetails$dbms diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index fcaaebb..1876a4c 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -135,11 +135,11 @@ addCharacterizationTargetIds <- function(settings){ studyPopulation <- unique(do.call('rbind', studyPopulationList)) %>% dplyr::group_by(dplyr::across(-dplyr::all_of(settingTypes))) %>% dplyr::summarise( - timeToEventSettings = any(.data$timeToEventSettings), - dechallengeRechallengeSettings = any(.data$dechallengeRechallengeSettings), - targetBaselineSettings = any(.data$targetBaselineSettings), - riskFactorSettings = any(.data$riskFactorSettings), - caseSeriesSettings = any(.data$caseSeriesSettings), + timeToEventSettings = any(.data$timeToEventSettings)*1, + dechallengeRechallengeSettings = any(.data$dechallengeRechallengeSettings)*1, + targetBaselineSettings = any(.data$targetBaselineSettings)*1, + riskFactorSettings = any(.data$riskFactorSettings)*1, + caseSeriesSettings = any(.data$caseSeriesSettings)*1, .groups = "drop" ) # give a new id called characterizationTargetIds per target and subset @@ -865,6 +865,28 @@ exportSharedObjects <- function( outputFolder = file.path(executionPath, 'target_attrition') ) + # NEW to improve extraction + # extract tte and dechal settings to T/O pairs + # ============== + tte_settings <- extractSettings(characterizationSettings, type = 'timeToEventSettings') + tte_settings$database_id <- databaseId + tte_settings$setting_id <- executionId + utils::write.csv( + x = formatDouble(tte_settings), + file = file.path(saveLocation, paste0(tablePrefix,'time_to_event_settings.csv')), + row.names = FALSE + ) + + dcrc_settings <- extractSettings(characterizationSettings, type = 'dechallengeRechallengeSettings') + dcrc_settings$database_id <- databaseId + dcrc_settings$setting_id <- executionId + utils::write.csv( + x = formatDouble(dcrc_settings), + file = file.path(saveLocation, paste0(tablePrefix,'dechallenge_rechallenge_settings.csv')), + row.names = FALSE + ) + # ============== + # export target settings table sql <- SqlRender::render( sql = "SELECT * FROM @target_settings_table;", @@ -1034,3 +1056,33 @@ exportSharedObjects <- function( } + +extractSettings <- function(characterizationSettings, type = 'timeToEventSettings'){ + settings <- characterizationSettings[[type]] + + if(!is.null(settings)){ + + extractedSettings <- unique(do.call( + what = 'rbind', + args = lapply( + X = settings, + FUN = function(x){ + expand.grid( + characterization_target_id = as.integer(x$characterizationTargetIds), + outcome_id = as.integer(x$outcomeIds) + ) + } + ))) + + return(extractedSettings) + + } else{ + return( + data.frame( + characterization_target_id = 0, + outcome_id = 0 + )) + } + +} + diff --git a/inst/settings/resultsDataModelSpecification.csv b/inst/settings/resultsDataModelSpecification.csv index f507d12..fcefd14 100644 --- a/inst/settings/resultsDataModelSpecification.csv +++ b/inst/settings/resultsDataModelSpecification.csv @@ -11,7 +11,7 @@ rechallenge_fail_case_series,database_id,varchar(100),Yes,Yes,No,No,The database rechallenge_fail_case_series,dechallenge_stop_interval,int,Yes,Yes,No,No,The time period that É rechallenge_fail_case_series,dechallenge_evaluation_window,int,Yes,Yes,No,No,The time period that É rechallenge_fail_case_series,characterization_target_id,bigint,Yes,Yes,No,No,The cohort definition id for the target cohort -rechallenge_fail_case_series,included,int,No,No,No,No,Whether this was included in the study population settings eras +rechallenge_fail_case_series,included,char(1),No,No,No,No,Whether this was included in the study population settings eras rechallenge_fail_case_series,outcome_cohort_definition_id,bigint,Yes,Yes,No,No,The cohort definition id for the outcome cohort rechallenge_fail_case_series,person_key,int,Yes,Yes,No,No,The dense rank for the patient (an identifier that is not the same as the database) rechallenge_fail_case_series,subject_id,bigint,No,No,No,No,The person identifier for the failed case series (optional) @@ -100,14 +100,14 @@ target_settings,min_prior_observation,int,No,No,No,No,"Target exposures with < m target_settings,nesting_cohort_id,bigint,No,No,No,No,The nesting id for the popualtion of interest target_settings,min_age,int,No,No,No,No,The min age to be includedfor the popualtion of interest target_settings,max_age,int,No,No,No,No,The max age to be includedfor the popualtion of interest -target_settings,study_start,date,No,No,No,No,The earliest date to be included for the popualtion of interest -target_settings,study_end,date,No,No,No,No,The latest date to be included for the popualtion of interest +target_settings,study_start,char(8),No,No,No,No,The earliest date to be included for the popualtion of interest +target_settings,study_end,char(8),No,No,No,No,The latest date to be included for the popualtion of interest target_settings,gender_concept_ids,varchar(100),No,No,No,No,The gender conept ids to be included for the popualtion of interest -target_settings,time_to_event_settings,bit,No,No,No,No,Whether used in time to event -target_settings,dechallenge_rechallenge_settings,bit,No,No,No,No,Whether used in dechal-rechal -target_settings,target_baseline_settings,bit,No,No,No,No,Whether used in target baseline -target_settings,risk_factor_settings,bit,No,No,No,No,Whether used in risk factor -target_settings,case_series_settings,bit,No,No,No,No,Whether used in case series +target_settings,time_to_event_settings,char(1),No,No,No,No,Whether used in time to event +target_settings,dechallenge_rechallenge_settings,char(1),No,No,No,No,Whether used in dechal-rechal +target_settings,target_baseline_settings,char(1),No,No,No,No,Whether used in target baseline +target_settings,risk_factor_settings,char(1),No,No,No,No,Whether used in risk factor +target_settings,case_series_settings,char(1),No,No,No,No,Whether used in case series case_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier case_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier case_settings,characterization_case_id,bigint,Yes,Yes,No,No,"The case cohort id that is unique per characterization_target_id, outcome_id, outcome_washout_days and time-at-risk settings" @@ -118,8 +118,8 @@ case_settings,start_anchor,varchar(15),No,No,No,No,The start anchor case_settings,end_anchor,varchar(15),No,No,No,No,The end anchor case_settings,risk_window_start,int,No,No,No,No,The risk window start case_settings,risk_window_end,int,No,No,No,No,The risk window end -case_settings,risk_factor_settings,varchar(50),No,No,No,No,Whether this case was used in risk-factor -case_settings,case_series_settings,varchar(50),No,No,No,No,Whether this case was used in case-series +case_settings,risk_factor_settings,char(1),No,No,No,No,Whether this case was used in risk-factor +case_settings,case_series_settings,char(1),No,No,No,No,Whether this case was used in case-series case_series_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier case_series_settings,case_pre_target_duration,int,No,No,No,No,The number of days before target index to create the before target period in case series case_series_settings,case_post_outcome_duration,int,No,No,No,No,The number of days after first outcome after target to create the after outcome period in case series @@ -225,3 +225,11 @@ case_attrition,n_events,bigint,No,No,No,No,The number of events remaining case_attrition,n_people,bigint,No,No,No,No,The number of people remaining case_attrition,database_id,varchar(100),Yes,Yes,No,No,The database identifier case_attrition,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +time_to_event_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +time_to_event_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier +time_to_event_settings,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort id +time_to_event_settings,outcome_id,bigint,Yes,Yes,No,No,The outcome cohort id +dechallenge_rechallenge_settings,setting_id,varchar(50),Yes,Yes,No,No,The run identifier +dechallenge_rechallenge_settings,database_id,varchar(100),Yes,Yes,No,No,The database identifier +dechallenge_rechallenge_settings,characterization_target_id,bigint,Yes,Yes,No,No,The characterization cohort id +dechallenge_rechallenge_settings,outcome_id,bigint,Yes,Yes,No,No,The outcome cohort id diff --git a/inst/sql/sql_server/UpdateVersionNumber.sql b/inst/sql/sql_server/UpdateVersionNumber.sql index e036e85..68740b9 100644 --- a/inst/sql/sql_server/UpdateVersionNumber.sql +++ b/inst/sql/sql_server/UpdateVersionNumber.sql @@ -1,5 +1,5 @@ {DEFAULT @package_version = package_version} -{DEFAULT @version_number = '3.0.0'} +{DEFAULT @version_number = '4.0.0'} DELETE FROM @database_schema.@table_prefix@package_version; INSERT INTO @database_schema.@table_prefix@package_version (version_number) VALUES ('@version_number'); diff --git a/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql b/inst/sql/sql_server/migrations/Migration_2-v4_0_0_table_change.sql similarity index 82% rename from inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql rename to inst/sql/sql_server/migrations/Migration_2-v4_0_0_table_change.sql index ed152ab..53b6be5 100644 --- a/inst/sql/sql_server/migrations/Migration_2-v3_1_0_table_change.sql +++ b/inst/sql/sql_server/migrations/Migration_2-v4_0_0_table_change.sql @@ -1,8 +1,3 @@ --- Database migrations for verion 0.3.0 --- This migration updates the schema: - -- 1. to store the charcterization version - -- 2. Add a migrations table for supporting database migrations - {DEFAULT @package_version = package_version} {DEFAULT @migration = migration} {DEFAULT @table_prefix = ''} @@ -116,21 +111,21 @@ ADD COLUMN min_age INT; ALTER TABLE @database_schema.@table_prefixtarget_settings ADD COLUMN max_age INT; ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN study_start DATE; +ADD COLUMN study_start CHAR(8); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN study_end DATE; +ADD COLUMN study_end CHAR(8); ALTER TABLE @database_schema.@table_prefixtarget_settings ADD COLUMN gender_concept_ids VARCHAR(100); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN time_to_event_settings BIT; +ADD COLUMN time_to_event_settings CHAR(1); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN dechallenge_rechallenge_settings BIT; +ADD COLUMN dechallenge_rechallenge_settings CHAR(1); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN target_baseline_settings BIT; +ADD COLUMN target_baseline_settings CHAR(1); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN risk_factor_settings BIT; +ADD COLUMN risk_factor_settings CHAR(1); ALTER TABLE @database_schema.@table_prefixtarget_settings -ADD COLUMN case_series_settings BIT; +ADD COLUMN case_series_settings CHAR(1); -- =========================== @@ -141,15 +136,33 @@ ADD COLUMN case_series_settings BIT; ALTER TABLE @database_schema.@table_prefixcase_settings DROP COLUMN runtype; -- add: risk_factor_settings varchar(50) / case_series_settings varchar(50) ALTER TABLE @database_schema.@table_prefixcase_settings -ADD COLUMN risk_factor_settings VARCHAR(50); +ADD COLUMN risk_factor_settings CHAR(1); ALTER TABLE @database_schema.@table_prefixcase_settings -ADD COLUMN case_series_settings VARCHAR(50); +ADD COLUMN case_series_settings CHAR(1); -- =========================== -- 8) Add included to rechallenge_fail_case_series -- =========================== ALTER TABLE @database_schema.@table_prefixrechallenge_fail_case_series -ADD COLUMN included INT; +ADD COLUMN included CHAR(1); + +-- =========================== +-- 8) Add new settings tables +-- =========================== +CREATE TABLE @database_schema.@table_prefixtime_to_event_settings( + setting_id VARCHAR(50), + database_id VARCHAR(100), + characterization_target_id BIGINT, + outcome_id BIGINT, + PRIMARY KEY (setting_id, database_id,characterization_target_id, outcome_id) +); +CREATE TABLE @database_schema.@table_prefixdechallenge_rechallenge_settings( + setting_id VARCHAR(50), + database_id VARCHAR(100), + characterization_target_id BIGINT, + outcome_id BIGINT, + PRIMARY KEY (setting_id, database_id,characterization_target_id, outcome_id) +); From 23e842b0a04afd4933623c46b4690c3694b7e7c1 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 9 Jul 2026 12:07:36 -0400 Subject: [PATCH 23/39] adding option to washout using outcomes outside observation period use restrictWashoutToObs = FALSE in createCharacterizationSettings to ignore observation period for washout --- R/CohortGeneration.R | 9 +++++++-- R/RunCharacterization.R | 7 +++++-- inst/sql/sql_server/NonCaseCohorts.sql | 7 +++++-- man/createCharacterizationSettings.Rd | 5 ++++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 15faf64..a379837 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -180,7 +180,9 @@ generateCohorts <- function( executionPath = executionPath, settings = ParallelLogger::convertJsonToSettings(cohortJobs$jobs[i,"settings"]), - jobId = cohortJobs$jobs[i, "jobId"] + jobId = cohortJobs$jobs[i, "jobId"], + + restrictWashoutToObs = characterizationSettings$restrictWashoutToObs ) } @@ -721,6 +723,7 @@ generateNonCases <- function( jobId, mode, incremental, + restrictWashoutToObs, ... ){ @@ -754,7 +757,9 @@ generateNonCases <- function( cohort_table = outcomeTable, use_plp = mode == 'PatientLevelPrediction', - use_ci = mode == 'CohortIncidence' + use_ci = mode == 'CohortIncidence', + + restrict_washout_to_obs = restrictWashoutToObs ) DatabaseConnector::executeSql( diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index a41413b..42cb5d0 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -10,6 +10,7 @@ #' @param targetBaselineSettings A list of targetBaselineSettings settings #' @param riskFactorSettings A list of riskFactorSettings settings #' @param caseSeriesSettings A list of caseSeriesSettings settings +#' @param restrictWashoutToObs Whether to restrict to outcomes in observation period for washout in risk factors #' @family LargeScale #' #' @return @@ -33,7 +34,8 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL + caseSeriesSettings = NULL, + restrictWashoutToObs = TRUE ) { errorMessages <- checkmate::makeAssertCollection() @@ -86,7 +88,8 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = dechallengeRechallengeSettings, targetBaselineSettings = targetBaselineSettings, riskFactorSettings = riskFactorSettings, - caseSeriesSettings = caseSeriesSettings + caseSeriesSettings = caseSeriesSettings, + restrictWashoutToObs = restrictWashoutToObs ) class(settings) <- "characterizationSettings" diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index b26721a..38e2184 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -40,8 +40,11 @@ SELECT LEFT JOIN @cohort_schema.@cohort_table o ON t.subject_id = o.subject_id AND case_settings.outcome_id = o.cohort_definition_id - AND o.cohort_start_date >= t.observation_period_start_date - AND o.cohort_start_date <= t.observation_period_end_date + + {@restrict_washout_to_obs}?{ + AND o.cohort_start_date >= t.observation_period_start_date + AND o.cohort_start_date <= t.observation_period_end_date + } -- outcome starts before TAR start AND o.cohort_start_date <= dateadd(day, @risk_window_start, t.@start_anchor_date) -- outcome end after washout prior before TAR start diff --git a/man/createCharacterizationSettings.Rd b/man/createCharacterizationSettings.Rd index b70f89a..f0f7199 100644 --- a/man/createCharacterizationSettings.Rd +++ b/man/createCharacterizationSettings.Rd @@ -9,7 +9,8 @@ createCharacterizationSettings( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL + caseSeriesSettings = NULL, + restrictWashoutToObs = TRUE ) } \arguments{ @@ -22,6 +23,8 @@ createCharacterizationSettings( \item{riskFactorSettings}{A list of riskFactorSettings settings} \item{caseSeriesSettings}{A list of caseSeriesSettings settings} + +\item{restrictWashoutToObs}{Whether to restrict to outcomes in observation period for washout in risk factors} } \value{ Returns the connection to the sqlite database From 338de6c01ef740f141605ae67a06a53a4d40c542 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 9 Jul 2026 22:24:04 -0400 Subject: [PATCH 24/39] Update NonCaseCohorts.sql --- inst/sql/sql_server/NonCaseCohorts.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index 38e2184..eb23181 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -18,6 +18,7 @@ SELECT MAX(CASE WHEN o.cohort_start_date IS NOT NULL AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) + -- Note: should it be > or >= ? this will remove overlapping even when washout is 0 AND o.cohort_end_date > dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) THEN 1 else 0 END) AS outcome_in_washout_before_tar, @@ -46,7 +47,7 @@ SELECT AND o.cohort_start_date <= t.observation_period_end_date } -- outcome starts before TAR start - AND o.cohort_start_date <= dateadd(day, @risk_window_start, t.@start_anchor_date) + AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) -- outcome end after washout prior before TAR start AND o.cohort_end_date >= dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) From 027f788771f22c665a1fafa514d75014f577d767 Mon Sep 17 00:00:00 2001 From: jreps Date: Thu, 9 Jul 2026 22:27:50 -0400 Subject: [PATCH 25/39] noncase edits - fixing outcome before TAR - adding option to do washout using outcomes outside of observation period --- R/CohortGeneration.R | 9 +++++++-- R/RunCharacterization.R | 7 +++++-- inst/sql/sql_server/NonCaseCohorts.sql | 4 +++- man/createCharacterizationSettings.Rd | 5 ++++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 09a0f15..9b9ad8a 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -200,7 +200,9 @@ generateCohorts <- function( executionPath = executionPath, settings = ParallelLogger::convertJsonToSettings(cohortJobs$jobs[i,"settings"]), - jobId = cohortJobs$jobs[i, "jobId"] + jobId = cohortJobs$jobs[i, "jobId"], + + restrictWashoutToObs = characterizationSettings$restrictWashoutToObs ) } @@ -712,6 +714,7 @@ generateNonCases <- function( jobId, mode, incremental, + restrictWashoutToObs, ... ){ @@ -746,7 +749,9 @@ generateNonCases <- function( cohort_table = outcomeTable, use_plp = mode == 'PatientLevelPrediction', - use_ci = mode == 'CohortIncidence' + use_ci = mode == 'CohortIncidence', + + restrict_washout_to_obs = restrictWashoutToObs ) DatabaseConnector::executeSql( diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index 1876a4c..b2d71f2 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -10,6 +10,7 @@ #' @param targetBaselineSettings A list of targetBaselineSettings settings #' @param riskFactorSettings A list of riskFactorSettings settings #' @param caseSeriesSettings A list of caseSeriesSettings settings +#' @param restrictWashoutToObs Whether the outcome washout can use outcomes outside the observation period when calling risk factor analysis #' @family LargeScale #' #' @return @@ -37,7 +38,8 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL + caseSeriesSettings = NULL, + restrictWashoutToObs = TRUE ) { errorMessages <- checkmate::makeAssertCollection() @@ -90,7 +92,8 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = dechallengeRechallengeSettings, targetBaselineSettings = targetBaselineSettings, riskFactorSettings = riskFactorSettings, - caseSeriesSettings = caseSeriesSettings + caseSeriesSettings = caseSeriesSettings, + restrictWashoutToObs = restrictWashoutToObs ) # update the settings replace the popSet with the characterizationTargetIds diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index 5a0d21a..2f0a734 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -43,10 +43,12 @@ SELECT LEFT JOIN @cohort_schema.@cohort_table o ON t.subject_id = o.subject_id AND case_settings.outcome_id = o.cohort_definition_id + {@restrict_washout_to_obs}?{ AND o.cohort_start_date >= t.observation_period_start_date AND o.cohort_start_date <= t.observation_period_end_date + } -- outcome starts before TAR start - AND o.cohort_start_date <= dateadd(day, @risk_window_start, t.@start_anchor_date) + AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) -- outcome end after washout prior before TAR start AND o.cohort_end_date >= dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) diff --git a/man/createCharacterizationSettings.Rd b/man/createCharacterizationSettings.Rd index 669af3a..7e1dc77 100644 --- a/man/createCharacterizationSettings.Rd +++ b/man/createCharacterizationSettings.Rd @@ -9,7 +9,8 @@ createCharacterizationSettings( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL + caseSeriesSettings = NULL, + restrictWashoutToObs = TRUE ) } \arguments{ @@ -22,6 +23,8 @@ createCharacterizationSettings( \item{riskFactorSettings}{A list of riskFactorSettings settings} \item{caseSeriesSettings}{A list of caseSeriesSettings settings} + +\item{restrictWashoutToObs}{Whether the outcome washout can use outcomes outside the observation period when calling risk factor analysis} } \value{ Returns the connection to the sqlite database From 7cf32ff13f8ffd34db68a6d90912f73c5d3c54e2 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 09:24:52 -0400 Subject: [PATCH 26/39] Update NonCaseCohorts.sql --- inst/sql/sql_server/NonCaseCohorts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index eb23181..3c387ab 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -19,7 +19,7 @@ SELECT WHEN o.cohort_start_date IS NOT NULL AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) -- Note: should it be > or >= ? this will remove overlapping even when washout is 0 - AND o.cohort_end_date > dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) + AND o.cohort_end_date >= dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) THEN 1 else 0 END) AS outcome_in_washout_before_tar, -- ADD has outcome in TAR (left join CASES on characterization_target_id, row_id and ) From 62a62a9939429cd5759ce007f33eb35d043182d3 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 09:27:07 -0400 Subject: [PATCH 27/39] Update NonCaseCohorts.sql --- inst/sql/sql_server/NonCaseCohorts.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index 2f0a734..ae964a4 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -21,7 +21,8 @@ SELECT MAX(CASE WHEN o.cohort_start_date IS NOT NULL AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) - AND o.cohort_end_date > dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) + -- Note: should this be >= or > ?? + AND o.cohort_end_date >= dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) THEN 1 else 0 END) AS outcome_in_washout_before_tar, -- ADD has outcome in TAR (left join CASES on characterization_target_id, row_id and ) From 26ea8ed7c89367aabdd078fbb10e58a133f8cb4d Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 10:57:55 -0400 Subject: [PATCH 28/39] fixing target/case code for situation where cohorts are not nicely ordered fixing target/case code for situation where cohorts are not nicely ordered --- inst/sql/sql_server/CaseCohorts.sql | 4 +++- inst/sql/sql_server/TargetCohorts.sql | 23 ++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/inst/sql/sql_server/CaseCohorts.sql b/inst/sql/sql_server/CaseCohorts.sql index 553bf54..752839f 100644 --- a/inst/sql/sql_server/CaseCohorts.sql +++ b/inst/sql/sql_server/CaseCohorts.sql @@ -23,7 +23,9 @@ FROM @characterization_schema.@characterization_table t INNER JOIN ( SELECT *, - ISNULL(datediff(day, LAG(cohort_end_date) OVER(partition BY subject_id, cohort_definition_id ORDER BY cohort_start_date ASC), cohort_start_date ), (@outcome_washout+1)) outcome_washout_time + -- NOTE this does not consider observation period + -- and no outcome erification if done currently + ISNULL(DATEDIFF(day, MAX(cohort_end_date) OVER (PARTITION BY subject_id, cohort_definition_id ORDER BY cohort_start_date ASC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), cohort_start_date), (@outcome_washout+1)) AS outcome_washout_time FROM @cohort_schema.@cohort_table WHERE cohort_definition_id IN (@outcome_cohort_ids) ) o diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index 29da51b..a41b9a5 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -24,12 +24,12 @@ INTO #temp_target FROM (SELECT cohort_definition_id, - --@limit_to_first_in_n_days AS limit_to_first_in_n_days, - --@min_prior_observation AS min_prior_observation, subject_id, cohort_start_date, cohort_end_date, - ISNULL(datediff(day, LAG(cohort_end_date) OVER(partition BY subject_id, cohort_definition_id ORDER BY cohort_start_date ASC), cohort_start_date ), -1) AS time_between + -- edited this so it works even when cohort period are not ordered nicely + ISNULL(DATEDIFF(day, MAX(cohort_end_date) OVER (PARTITION BY subject_id, cohort_definition_id ORDER BY cohort_start_date ASC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), cohort_start_date), (@limit_to_first_in_n_days+1)) AS time_between + FROM @cohort_schema.@cohort_table WHERE cohort_definition_id IN (@cohort_ids) ) temp_cohort @@ -62,7 +62,7 @@ ON temp_cohort.cohort_definition_id = target_settings.target_id; SELECT * INTO #temp_target_first FROM #temp_target temp_cohort -WHERE (temp_cohort.time_between >= @limit_to_first_in_n_days OR temp_cohort.time_between = -1); +WHERE (temp_cohort.time_between >= @limit_to_first_in_n_days); -- now min prior obs SELECT * @@ -123,7 +123,20 @@ FROM #temp_target_age; -- finally date: {@study_start != '' | @study_end != ''}?{ -SELECT * +SELECT +cohort_definition_id, +row_number, +subject_id, +cohort_start_date, +-- edit the end date if after study end +{@study_end != ''}?{ +CASE WHEN @study_end < cohort_end_date THEN @study_end ELSE cohort_end_date END as cohort_end_date, +} : +{cohort_end_date,} +observation_period_start_date, +observation_period_end_date, +time_between, +char_type INTO #temp_target_date FROM #temp_target_gender WHERE 1 = 1 From 397837b24d84438afa16a8f98b89f938cd43d65a Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 11:12:41 -0400 Subject: [PATCH 29/39] fixing shiny --- R/ViewShiny.R | 26 +++++++++++++------------- inst/shinyConfigUpdate.json | 4 ++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 196cd80..0870a46 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -182,16 +182,10 @@ prepareCharacterizationShiny <- function( ) ) - - if(length(cohortIds) == 0){ - # if no cohortids then no results to view - return(invisible(list())) - } - DatabaseConnector::insertTable( connection = con, databaseSchema = "main", - tableName = "cg_COHORT_DEFINITION", + tableName = "cg_cohort_definition", data = data.frame( cohortDefinitionId = cohortIds, cohortName = getCohortNames(cohortIds, cohortDefinitionSet), @@ -201,17 +195,23 @@ prepareCharacterizationShiny <- function( ), camelCaseToSnakeCase = TRUE ) + } - databaseIds <- DatabaseConnector::querySql( + # add the new subset table + if (!"cg_cohort_subset_definition" %in% tables) { + DatabaseConnector::insertTable( connection = con, - sql = paste0("select distinct DATABASE_ID from main.DATABASE_META_DATA;"), - snakeCaseToCamelCase = TRUE - )$databaseId - + databaseSchema = "main", + tableName = "cg_cohort_subset_definition", + data = data.frame( + subsetDefinitionId = 1, + json = '{}' + ), + camelCaseToSnakeCase = TRUE + ) } - # create the settings for the database databaseSettings <- list( connectionDetailsSettings = list( diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json index 13e18d9..f5a751a 100644 --- a/inst/shinyConfigUpdate.json +++ b/inst/shinyConfigUpdate.json @@ -5,6 +5,8 @@ "tabName": "About", "tabText": "About", "shinyModulePackage": "OhdsiShinyModules", + "shinyModulePackageVersion": "v2.2.0", + "installSource": "CRAN", "uiFunction": "aboutViewer", "serverFunction": "aboutServer", "infoBoxFile": "aboutHelperFile()", @@ -16,6 +18,8 @@ "tabName": "Characterization", "tabText": "Characterization", "shinyModulePackage": "OhdsiShinyModules", + "shinyModulePackageVersion": "v2.2.0", + "installSource": "CRAN", "uiFunction": "characterizationViewer", "serverFunction": "characterizationServer", "infoBoxFile": "characterizationHelperFile()", From 5104fda70f4802523208e7a2474418b7d771d463 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 11:39:37 -0400 Subject: [PATCH 30/39] Update shinyConfigUpdate.json --- inst/shinyConfigUpdate.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json index f5a751a..c5932f6 100644 --- a/inst/shinyConfigUpdate.json +++ b/inst/shinyConfigUpdate.json @@ -6,7 +6,8 @@ "tabText": "About", "shinyModulePackage": "OhdsiShinyModules", "shinyModulePackageVersion": "v2.2.0", - "installSource": "CRAN", + "installSource": "github", + "gitHubRepo": "ohdsi", "uiFunction": "aboutViewer", "serverFunction": "aboutServer", "infoBoxFile": "aboutHelperFile()", @@ -19,7 +20,8 @@ "tabText": "Characterization", "shinyModulePackage": "OhdsiShinyModules", "shinyModulePackageVersion": "v2.2.0", - "installSource": "CRAN", + "installSource": "github", + "gitHubRepo": "ohdsi", "uiFunction": "characterizationViewer", "serverFunction": "characterizationServer", "infoBoxFile": "characterizationHelperFile()", From 18118e06d6547766bed6972bef1b23da43cf2064 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 11:44:43 -0400 Subject: [PATCH 31/39] Update shinyConfigUpdate.json --- inst/shinyConfigUpdate.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json index c5932f6..dc6965f 100644 --- a/inst/shinyConfigUpdate.json +++ b/inst/shinyConfigUpdate.json @@ -5,7 +5,7 @@ "tabName": "About", "tabText": "About", "shinyModulePackage": "OhdsiShinyModules", - "shinyModulePackageVersion": "v2.2.0", + "shinyModulePackageVersion": "2.2.0", "installSource": "github", "gitHubRepo": "ohdsi", "uiFunction": "aboutViewer", @@ -19,7 +19,7 @@ "tabName": "Characterization", "tabText": "Characterization", "shinyModulePackage": "OhdsiShinyModules", - "shinyModulePackageVersion": "v2.2.0", + "shinyModulePackageVersion": "2.2.0", "installSource": "github", "gitHubRepo": "ohdsi", "uiFunction": "characterizationViewer", From d8c5831a1eb86a9dad3c0c08026151851f0e9a33 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 11:56:53 -0400 Subject: [PATCH 32/39] Update ViewShiny.R --- R/ViewShiny.R | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 0870a46..443cd6d 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -211,6 +211,79 @@ prepareCharacterizationShiny <- function( ) } + # add the new cohort count + if (!"cg_cohort_count" %in% tables) { + + dbIds <- unique( + c( + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "analysis_ref;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "dechallenge_rechallenge;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$databaseId + ) + ) + + cohortIds <- unique( + c( + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_ID from ", tablePrefix, csvTablePrefix, "target_settings;"), + snakeCaseToCamelCase = TRUE + )$targetCohortId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_ID from ", tablePrefix, csvTablePrefix, "case_settings;"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$targetCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), + snakeCaseToCamelCase = TRUE + )$targetCohortDefinitionId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_COHORT_DEFINITION_ID from ", tablePrefix, csvTablePrefix, "rechallenge_fail_case_series;"), + snakeCaseToCamelCase = TRUE + )$outcomeCohortDefinitionId + ) + ) + + DatabaseConnector::insertTable( + connection = con, + databaseSchema = "main", + tableName = "cg_cohort_count", + data = data.frame( + cohortDefinitionId = cohortIds, + cohortId = cohortIds, + cohortEntries = rep(1000, length(cohortIds)), # fake + cohortSubjects = rep(1000, length(cohortIds)), # fake + databaseId = dbIds + ), + camelCaseToSnakeCase = TRUE + ) + } + # create the settings for the database databaseSettings <- list( From df3412e264617e3005d97160599de517fc742f3a Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 10 Jul 2026 15:50:01 -0400 Subject: [PATCH 33/39] outcome eras and shiny app fix - replacing row_number with row_id since row_number is a SQL function - adding code to do outcome eras for risk factor analysis - fixing shiny app in Characterization --- DESCRIPTION | 2 +- R/CaseSeries.R | 2 +- R/CohortGeneration.R | 112 ++++++++++++++-- R/RiskFactorAnalysis.R | 2 +- R/RunCharacterization.R | 8 +- R/ViewShiny.R | 70 +++++++++- inst/shinyConfigUpdate.json | 6 + inst/sql/sql_server/CaseCohorts.sql | 18 +-- .../sql_server/CreateTargetCohortTable.sql | 12 +- inst/sql/sql_server/DropTargetCohortTable.sql | 5 + inst/sql/sql_server/NonCaseCohorts.sql | 25 ++-- inst/sql/sql_server/OutcomeEras.sql | 58 +++++++++ inst/sql/sql_server/TargetCohorts.sql | 10 +- man/Characterization-package.Rd | 1 + man/cleanIncremental.Rd | 4 +- man/cleanNonIncremental.Rd | 4 +- man/createCaseSeriesSettings.Rd | 6 +- man/createCharacterizationSettings.Rd | 13 +- man/createCharacterizationTables.Rd | 6 +- man/createDuringCovariateSettings.Rd | 4 +- man/createRiskFactorSettings.Rd | 6 +- man/createSqliteDatabase.Rd | 6 +- man/createTargetBaselineSettings.Rd | 6 +- man/getDbDuringCovariateData.Rd | 4 +- man/insertResultsToDatabase.Rd | 6 +- man/loadCharacterizationSettings.Rd | 8 +- man/runCharacterizationAnalyses.Rd | 8 +- man/saveCharacterizationSettings.Rd | 8 +- tests/testthat/test-CaseSeries.R | 6 +- tests/testthat/test-CohortGeneration.R | 122 +++++++++++++++++- 30 files changed, 448 insertions(+), 100 deletions(-) create mode 100644 inst/sql/sql_server/OutcomeEras.sql diff --git a/DESCRIPTION b/DESCRIPTION index 0650416..4441daf 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,6 @@ Suggests: shiny, withr NeedsCompilation: no -RoxygenNote: 7.3.2 Encoding: UTF-8 VignetteBuilder: knitr +Config/roxygen2/version: 8.0.0 diff --git a/R/CaseSeries.R b/R/CaseSeries.R index c419a7b..6208ce6 100644 --- a/R/CaseSeries.R +++ b/R/CaseSeries.R @@ -234,7 +234,7 @@ computeCaseSeriesAnalyses <- function( cohortIds = c(caseIds$characterizationCaseId*10+3, caseIds$characterizationCaseId*10+4, caseIds$characterizationCaseId*10+5), - rowIdField = 'row_number', + rowIdField = 'row_id', covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettings), aggregated = TRUE, minCharacterizationMean = minCharacterizationMean, diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 9b9ad8a..cd7a907 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -27,6 +27,7 @@ generateCohorts <- function( # tables names characterizationTableWithHash <- paste0(outputTable, '_',settingHash, '_', dbHash) + outcomeEraTableWithHash <- paste0('outcome_era', '_',settingHash, '_', dbHash) targetSettingsTableWithHash <- paste0('target_settings', '_',settingHash, '_', dbHash) targetAttritionTableWithHash <- paste0('target_attrition', '_',settingHash, '_', dbHash) @@ -106,7 +107,8 @@ generateCohorts <- function( target_attrition_table = targetAttritionTableWithHash, target_count_table = targetCountTableWithHash, case_attrition_table = caseAttritionTableWithHash, - case_count_table = caseCountTableWithHash + case_count_table = caseCountTableWithHash, + outcome_era_table = outcomeEraTableWithHash ) DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) @@ -149,7 +151,8 @@ generateCohorts <- function( target_attrition_table = targetAttritionTableWithHash, target_count_table = targetCountTableWithHash, case_attrition_table = caseAttritionTableWithHash, - case_count_table = caseCountTableWithHash + case_count_table = caseCountTableWithHash, + outcome_era_table = outcomeEraTableWithHash ) DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) @@ -202,7 +205,7 @@ generateCohorts <- function( settings = ParallelLogger::convertJsonToSettings(cohortJobs$jobs[i,"settings"]), jobId = cohortJobs$jobs[i, "jobId"], - restrictWashoutToObs = characterizationSettings$restrictWashoutToObs + outcomeEraTable = outcomeEraTableWithHash ) } @@ -225,7 +228,8 @@ return(list( targetAttritionTable = targetAttritionTableWithHash, caseAttritionTable = caseAttritionTableWithHash, targetCountTable = targetCountTableWithHash, - caseCountTable = caseCountTableWithHash + caseCountTable = caseCountTableWithHash, + outcomeEraTable = outcomeEraTableWithHash ) ) } @@ -477,6 +481,7 @@ getCohortJobs <- function( if(mode != 'Efficient'){ + if(max(coi$riskFactorSettings[ind]) == 1){ nNonCase <- nNonCase + 1 jobs <- rbind(jobs, data.frame( @@ -503,6 +508,27 @@ getCohortJobs <- function( } + + # add in job for outcome eras per washout + # only run if there are cases and mode is not Efficient + # since efficient mode doesnt need the outcomes + if(!is.null(nrow(cases))){ + if(mode != 'Efficient'){ + ooi <- unique(cases[, c('outcomeId', 'outcomeWashoutDays')]) + for(outcomeWashoutDay in unique(ooi$outcomeWashoutDays)){ + jobs <- rbind(jobs, data.frame( + functionName = 'generateOutcomeEras', + settings = as.character(ParallelLogger::convertSettingsToJson(list( + outcomeIds = ooi$outcomeId[ooi$outcomeWashoutDays == outcomeWashoutDay], + outcomeWashoutDays = outcomeWashoutDay + ) + )), + jobId = paste("outcome_eras",i,outcomeWashoutDay, sep = "_") + )) + } + } + } + # removing nTargetJobs if(!is.null(nrow(targets))){ targets <- targets %>% dplyr::select(-"nTargetJobs") @@ -698,6 +724,7 @@ generateNonCases <- function( connectionDetails, cdmDatabaseSchema, characterizationTable, + outcomeEraTable, caseAttritionTable, caseCountTable, targetSettingsTable, @@ -714,7 +741,6 @@ generateNonCases <- function( jobId, mode, incremental, - restrictWashoutToObs, ... ){ @@ -739,6 +765,7 @@ generateNonCases <- function( outcome_cohort_ids = paste0(settings$outcomeIds, collapse = ','), characterization_target_ids = paste0(settings$characterizationTargetIds, collapse = ','), + outcome_era_table = outcomeEraTable, outcome_washout = settings$outcomeWashoutDays, risk_window_start = settings$riskWindowStart, start_anchor = settings$startAnchor, @@ -749,9 +776,7 @@ generateNonCases <- function( cohort_table = outcomeTable, use_plp = mode == 'PatientLevelPrediction', - use_ci = mode == 'CohortIncidence', - - restrict_washout_to_obs = restrictWashoutToObs + use_ci = mode == 'CohortIncidence' ) DatabaseConnector::executeSql( @@ -777,6 +802,73 @@ generateNonCases <- function( return(invisible(TRUE)) } +generateOutcomeEras <- function( + connectionDetails, + cdmDatabaseSchema, + characterizationTable, + caseAttritionTable, + caseCountTable, + targetSettingsTable, + caseSettingsTable, + characterizationDatabaseSchema, + tempEmulationSchema, + targetDatabaseSchema, + targetTable, + outcomeDatabaseSchema, + outcomeTable, + progressBar = interactive(), + executionPath, + settings, + jobId, + mode, + incremental, + outcomeEraTable, + ... +){ + + message(paste("Creating outcome eras for washout ", settings$outcomeWashoutDays)) + start <- Sys.time() + + connection <- DatabaseConnector::connect(connectionDetails) + on.exit(DatabaseConnector::disconnect(connection)) + + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = 'OutcomeEras.sql', + packageName = 'Characterization', + dbms = attributes(connection)$dbms, + tempEmulationSchema = tempEmulationSchema, + characterization_schema = characterizationDatabaseSchema, + outcome_era_table = outcomeEraTable, + outcome_ids = paste0(settings$outcomeIds, collapse = ','), + outcome_washout = settings$outcomeWashoutDays, + cohort_schema = outcomeDatabaseSchema, + cohort_table = outcomeTable + ) + + DatabaseConnector::executeSql( + connection = connection, + sql = sql, + progressBar = progressBar, + reportOverallTime = FALSE + ) + completionTime <- Sys.time() - start + + if(incremental){ + readr::write_csv( + file = file.path(executionPath,'cohort_job_tracker.csv'), + x = data.frame( + jobId = jobId, + completeDate = date() + ), + append = TRUE + ) + } + message(paste0("Creating Outcome Eras: took ", round(completionTime, digits = 1), " ", units(completionTime))) + + return(invisible(TRUE)) + +} + dropCohorts <- function( @@ -803,6 +895,7 @@ dropCohorts <- function( caseAttritionTableWithHash <- paste0('case_attrition', '_',settingHash, '_', dbHash) targetCountTableWithHash <- paste0('target_count', '_',settingHash, '_', dbHash) caseCountTableWithHash <- paste0('case_count', '_',settingHash, '_', dbHash) + outcomeEraTableWithHash <- paste0('outcome_era', '_',settingHash, '_', dbHash) sql <- SqlRender::loadRenderTranslateSql( @@ -817,7 +910,8 @@ dropCohorts <- function( target_count_table = targetCountTableWithHash, case_count_table = caseCountTableWithHash, target_settings_table = targetSettingsTableWithHash, - case_settings_table = caseSettingsTableWithHash + case_settings_table = caseSettingsTableWithHash, + outcome_era_table = outcomeEraTableWithHash ) DatabaseConnector::executeSql(connection, sql, progressBar = progressBar) diff --git a/R/RiskFactorAnalysis.R b/R/RiskFactorAnalysis.R index e9a2012..82ea289 100644 --- a/R/RiskFactorAnalysis.R +++ b/R/RiskFactorAnalysis.R @@ -259,7 +259,7 @@ computeRiskFactorAnalyses <- function( cohortTable = characterizationTable, cohortDatabaseSchema = characterizationDatabaseSchema, cohortIds = cohortIds, - rowIdField = 'row_number', + rowIdField = 'row_id', covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettings), aggregated = TRUE, minCharacterizationMean = minCharacterizationMean, diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index b2d71f2..7f293d3 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -10,7 +10,6 @@ #' @param targetBaselineSettings A list of targetBaselineSettings settings #' @param riskFactorSettings A list of riskFactorSettings settings #' @param caseSeriesSettings A list of caseSeriesSettings settings -#' @param restrictWashoutToObs Whether the outcome washout can use outcomes outside the observation period when calling risk factor analysis #' @family LargeScale #' #' @return @@ -38,8 +37,7 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL, - restrictWashoutToObs = TRUE + caseSeriesSettings = NULL ) { errorMessages <- checkmate::makeAssertCollection() @@ -92,8 +90,7 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = dechallengeRechallengeSettings, targetBaselineSettings = targetBaselineSettings, riskFactorSettings = riskFactorSettings, - caseSeriesSettings = caseSeriesSettings, - restrictWashoutToObs = restrictWashoutToObs + caseSeriesSettings = caseSeriesSettings ) # update the settings replace the popSet with the characterizationTargetIds @@ -587,6 +584,7 @@ runCharacterizationAnalyses <- function( # new inputs characterizationDatabaseSchema = outputDatabaseSchema, characterizationTable = tableNames$characterizationTable, + outcomeEraTable = tableNames$outcomeEraTable, targetSettingsTable = tableNames$targetSettingsTable, caseSettingsTable = tableNames$caseSettingsTable, diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 69ff1f0..58c94ef 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -184,12 +184,74 @@ prepareCharacterizationShiny <- function( camelCaseToSnakeCase = TRUE ) - databaseIds <- DatabaseConnector::querySql( + } + + + # add the new subset table + if (!"cg_cohort_subset_definition" %in% tables) { + DatabaseConnector::insertTable( connection = con, - sql = paste0("select distinct DATABASE_ID from main.DATABASE_META_DATA;"), - snakeCaseToCamelCase = TRUE - )$databaseId + databaseSchema = "main", + tableName = "cg_cohort_subset_definition", + data = data.frame( + subsetDefinitionId = 1, + json = '{}' + ), + camelCaseToSnakeCase = TRUE + ) + } + + # add the new cohort count + if (!"cg_cohort_count" %in% tables) { + + dbIds <- unique( + c( + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "analysis_ref;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "dechallenge_rechallenge;"), + snakeCaseToCamelCase = TRUE + )$databaseId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct DATABASE_ID from ", tablePrefix, csvTablePrefix, "time_to_event;"), + snakeCaseToCamelCase = TRUE + )$databaseId + ) + ) + + cohortIds <- unique( + c( + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct TARGET_ID from ", tablePrefix, csvTablePrefix, "target_settings;"), + snakeCaseToCamelCase = TRUE + )$targetId, + DatabaseConnector::querySql( + connection = con, + sql = paste0("select distinct OUTCOME_ID from ", tablePrefix, csvTablePrefix, "case_settings;"), + snakeCaseToCamelCase = TRUE + )$outcomeId + ) + ) + DatabaseConnector::insertTable( + connection = con, + databaseSchema = "main", + tableName = "cg_cohort_count", + data = data.frame( + cohortDefinitionId = cohortIds, + cohortId = cohortIds, + cohortEntries = rep(1000, length(cohortIds)), # fake + cohortSubjects = rep(1000, length(cohortIds)), # fake + databaseId = dbIds + ), + camelCaseToSnakeCase = TRUE + ) } diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json index 13e18d9..dc6965f 100644 --- a/inst/shinyConfigUpdate.json +++ b/inst/shinyConfigUpdate.json @@ -5,6 +5,9 @@ "tabName": "About", "tabText": "About", "shinyModulePackage": "OhdsiShinyModules", + "shinyModulePackageVersion": "2.2.0", + "installSource": "github", + "gitHubRepo": "ohdsi", "uiFunction": "aboutViewer", "serverFunction": "aboutServer", "infoBoxFile": "aboutHelperFile()", @@ -16,6 +19,9 @@ "tabName": "Characterization", "tabText": "Characterization", "shinyModulePackage": "OhdsiShinyModules", + "shinyModulePackageVersion": "2.2.0", + "installSource": "github", + "gitHubRepo": "ohdsi", "uiFunction": "characterizationViewer", "serverFunction": "characterizationServer", "infoBoxFile": "characterizationHelperFile()", diff --git a/inst/sql/sql_server/CaseCohorts.sql b/inst/sql/sql_server/CaseCohorts.sql index 752839f..f66261e 100644 --- a/inst/sql/sql_server/CaseCohorts.sql +++ b/inst/sql/sql_server/CaseCohorts.sql @@ -6,7 +6,7 @@ IF OBJECT_ID('tempdb..#characterization_cases', 'U') IS NOT NULL DROP TABLE #cha SELECT case_settings.characterization_case_id as cohort_definition_id, -t.row_number, +t.row_id, t.subject_id, t.cohort_start_date, t.cohort_end_date, @@ -48,13 +48,13 @@ AND o.cohort_start_date >= t.observation_period_start_date AND o.cohort_start_date <= t.observation_period_end_date -- outcome starts before TAR end AND o.cohort_start_date <= dateadd(day, @risk_window_end, t.@end_anchor_date) --- outcome starts (ends?) after TAR start +-- outcome starts after TAR start AND o.cohort_start_date >= dateadd(day, @risk_window_start, t.@start_anchor_date) -- make sure to only get first outcome date during TAR GROUP BY case_settings.characterization_case_id, -t.row_number, +t.row_id, t.subject_id, t.cohort_start_date, t.cohort_end_date, @@ -71,11 +71,11 @@ WHERE char_type = 'cases' AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id*10+1 FROM #characterization_cases); INSERT INTO @characterization_schema.@characterization_table( -cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, char_type +cohort_definition_id, row_id, subject_id, cohort_start_date, cohort_end_date, char_type ) SELECT CAST(cohort_definition_id*10+1 as BIGINT), -row_number, +row_id, subject_id, cohort_start_date, cohort_end_date, @@ -97,11 +97,11 @@ SELECT cohort_definition_id*10+5 FROM #characterization_cases INSERT INTO @characterization_schema.@characterization_table( -cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, char_type +cohort_definition_id, row_id, subject_id, cohort_start_date, cohort_end_date, char_type ) SELECT CAST(cohort_definition_id*10+3 as BIGINT), -row_number, +row_id, subject_id, DATEADD(day, -@case_series_before, cohort_start_date), DATEADD(day, 0, cohort_start_date), @@ -113,7 +113,7 @@ UNION SELECT CAST(cohort_definition_id*10+4 as BIGINT), -row_number, +row_id, subject_id, DATEADD(day, 1, cohort_start_date), DATEADD(day, 0, outcome_start_date), @@ -124,7 +124,7 @@ UNION SELECT CAST(cohort_definition_id*10+5 as BIGINT), -row_number, +row_id, subject_id, DATEADD(day, 1, outcome_start_date), DATEADD(day, @case_series_after, outcome_end_date), diff --git a/inst/sql/sql_server/CreateTargetCohortTable.sql b/inst/sql/sql_server/CreateTargetCohortTable.sql index 29f584d..57f45b8 100644 --- a/inst/sql/sql_server/CreateTargetCohortTable.sql +++ b/inst/sql/sql_server/CreateTargetCohortTable.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS @characterization_schema.@characterization_table; CREATE TABLE @characterization_schema.@characterization_table( cohort_definition_id BIGINT, -row_number BIGINT, +row_id BIGINT, subject_id BIGINT, cohort_start_date DATE, cohort_end_date DATE, @@ -44,3 +44,13 @@ cohort_type VARCHAR(10), n_events BIGINT, n_people BIGINT ); + +-- outcome era table +DROP TABLE IF EXISTS @characterization_schema.@outcome_era_table; +CREATE TABLE @characterization_schema.@outcome_era_table( +cohort_definition_id BIGINT, +outcome_washout BIGINT, +subject_id BIGINT, +cohort_start_date DATE, +cohort_end_date DATE +); diff --git a/inst/sql/sql_server/DropTargetCohortTable.sql b/inst/sql/sql_server/DropTargetCohortTable.sql index 2dee2c5..03cfd15 100644 --- a/inst/sql/sql_server/DropTargetCohortTable.sql +++ b/inst/sql/sql_server/DropTargetCohortTable.sql @@ -2,6 +2,8 @@ {DEFAULT @drop_char_counts = true} {DEFAULT @drop_char_attr = true} {DEFAULT @drop_char_settings = true} +{DEFAULT @drop_outcome_era = true} + {@drop_char_cohorts}?{ DROP TABLE IF EXISTS @characterization_schema.@characterization_table; @@ -22,3 +24,6 @@ DROP TABLE IF EXISTS @characterization_schema.@target_settings_table; DROP TABLE IF EXISTS @characterization_schema.@case_settings_table; } +{@drop_outcome_era}?{ +DROP TABLE IF EXISTS @characterization_schema.@outcome_era_table; +} diff --git a/inst/sql/sql_server/NonCaseCohorts.sql b/inst/sql/sql_server/NonCaseCohorts.sql index ae964a4..fdb129c 100644 --- a/inst/sql/sql_server/NonCaseCohorts.sql +++ b/inst/sql/sql_server/NonCaseCohorts.sql @@ -6,7 +6,7 @@ IF OBJECT_ID('tempdb..#temp_non_cases_pass_washout', 'U') IS NOT NULL DROP TABLE SELECT case_settings.characterization_case_id*10+2 as cohort_definition_id, - t.row_number, + t.row_id, t.subject_id, t.cohort_start_date, t.cohort_end_date, @@ -26,7 +26,7 @@ SELECT THEN 1 else 0 END) AS outcome_in_washout_before_tar, -- ADD has outcome in TAR (left join CASES on characterization_target_id, row_id and ) - MAX(CASE WHEN cases.row_number IS NOT NULL THEN 1 else 0 END) AS outcome_during_tar + MAX(CASE WHEN cases.row_id IS NOT NULL THEN 1 else 0 END) AS outcome_during_tar INTO #temp_non_cases FROM @characterization_schema.@characterization_table t @@ -41,23 +41,24 @@ SELECT ) case_settings ON t.cohort_definition_id = case_settings.characterization_target_id - LEFT JOIN @cohort_schema.@cohort_table o + -- EDITED to collapse using outcome washout + LEFT JOIN @characterization_schema.@outcome_era_table o + --LEFT JOIN @cohort_schema.@cohort_table o + ON t.subject_id = o.subject_id AND case_settings.outcome_id = o.cohort_definition_id - {@restrict_washout_to_obs}?{ - AND o.cohort_start_date >= t.observation_period_start_date - AND o.cohort_start_date <= t.observation_period_end_date - } + AND o.outcome_washout = @outcome_washout + -- outcome starts before TAR start AND o.cohort_start_date < dateadd(day, @risk_window_start, t.@start_anchor_date) -- outcome end after washout prior before TAR start AND o.cohort_end_date >= dateadd(day, -@outcome_washout, dateadd(day, @risk_window_start, t.@start_anchor_date)) - -- use real table not temp case table? + -- join to cases LEFT JOIN (SELECT * from @characterization_schema.@characterization_table WHERE char_type = 'cases' ) cases ON cases.cohort_definition_id = case_settings.characterization_case_id*10+1 - AND cases.row_number = t.row_number + AND cases.row_id = t.row_id WHERE case_settings.outcome_id IN (@outcome_cohort_ids) AND case_settings.characterization_target_id IN (@characterization_target_ids) @@ -65,7 +66,7 @@ SELECT GROUP BY case_settings.characterization_case_id, - t.row_number, + t.row_id, t.subject_id, t.cohort_start_date, t.cohort_end_date; @@ -78,12 +79,12 @@ AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_non -- now determine the non-cases INSERT INTO @characterization_schema.@characterization_table( - cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, char_type + cohort_definition_id, row_id, subject_id, cohort_start_date, cohort_end_date, char_type ) SELECT temp.cohort_definition_id, - temp.row_number, + temp.row_id, temp.subject_id, temp.cohort_start_date, temp.cohort_end_date, diff --git a/inst/sql/sql_server/OutcomeEras.sql b/inst/sql/sql_server/OutcomeEras.sql new file mode 100644 index 0000000..028031e --- /dev/null +++ b/inst/sql/sql_server/OutcomeEras.sql @@ -0,0 +1,58 @@ +-- remove existing results +DELETE FROM @characterization_schema.@outcome_era_table +WHERE cohort_definition_id in (@outcome_ids) +AND outcome_washout = @outcome_washout +; + +-- now determine the non-cases + INSERT INTO @characterization_schema.@outcome_era_table( + cohort_definition_id, outcome_washout, subject_id, cohort_start_date, cohort_end_date + ) + +SELECT + cohort_definition_id, + @outcome_washout as outcome_washout, + subject_id, + MIN(cohort_start_date) AS cohort_start_date, + MAX(cohort_end_date) AS cohort_end_date + +FROM ( + + SELECT + cohort_definition_id, + subject_id, + cohort_start_date, + cohort_end_date, + + SUM( + CASE WHEN previous_cohort_end_date >= cohort_start_date THEN 0 + ELSE 1 + END) + OVER ( + PARTITION BY cohort_definition_id, subject_id + ORDER BY cohort_start_date + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS era_count + + FROM ( + SELECT + cohort_definition_id, + subject_id, + cohort_start_date, + cohort_end_date, + MAX(DATEADD(day, @outcome_washout, cohort_end_date)) OVER ( + PARTITION BY cohort_definition_id, subject_id + ORDER BY cohort_start_date ASC + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ) AS previous_cohort_end_date + FROM @cohort_schema.@cohort_table + + WHERE cohort_definition_id IN (@outcome_ids) + ) prior_eras + +) temp + +GROUP BY +cohort_definition_id, +subject_id, +era_count; diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index a41b9a5..a814906 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -11,7 +11,7 @@ IF OBJECT_ID('tempdb..#temp_target_date', 'U') IS NOT NULL DROP TABLE #temp_targ -- ========================= SELECT CAST(target_settings.characterization_target_id AS BIGINT) AS cohort_definition_id, -row_number() over(PARTITION BY CAST(target_settings.characterization_target_id AS BIGINT) ORDER BY temp_cohort.subject_id, temp_cohort.cohort_start_date ASC) AS row_number, +row_number() over(PARTITION BY CAST(target_settings.characterization_target_id AS BIGINT) ORDER BY temp_cohort.subject_id, temp_cohort.cohort_start_date ASC) AS row_id, temp_cohort.subject_id, temp_cohort.cohort_start_date, temp_cohort.cohort_end_date, @@ -73,7 +73,7 @@ WHERE datediff(day, observation_period_start_date, cohort_start_date) >= @min_pr -- now nesting {@nesting_cohort_id != 0}?{SELECT t.cohort_definition_id, -t.row_number, +t.row_id, t.subject_id, t.cohort_start_date, -- use the nesting end date if it is before the target end date @@ -125,7 +125,7 @@ FROM #temp_target_age; {@study_start != '' | @study_end != ''}?{ SELECT cohort_definition_id, -row_number, +row_id, subject_id, cohort_start_date, -- edit the end date if after study end @@ -163,13 +163,13 @@ AND cohort_definition_id in (SELECT DISTINCT cohort_definition_id FROM #temp_tar -- insert the new rows -- now determine the non-cases INSERT INTO @characterization_schema.@characterization_table( - cohort_definition_id, row_number, subject_id, cohort_start_date, cohort_end_date, + cohort_definition_id, row_id, subject_id, cohort_start_date, cohort_end_date, observation_period_start_date, observation_period_end_date, char_type ) SELECT temp.cohort_definition_id, - temp.row_number, + temp.row_id, temp.subject_id, temp.cohort_start_date, temp.cohort_end_date, -- TODO: update cohort_end_date to be study_end_date if study_end_date is before? diff --git a/man/Characterization-package.Rd b/man/Characterization-package.Rd index 1315097..9b0573a 100644 --- a/man/Characterization-package.Rd +++ b/man/Characterization-package.Rd @@ -22,6 +22,7 @@ Useful links: Authors: \itemize{ + \item Jenna Reps \email{jreps@its.jnj.com} \item Patrick Ryan \email{ryan@ohdsi.org} \item Chris Knoll \email{knoll@ohdsi.org} } diff --git a/man/cleanIncremental.Rd b/man/cleanIncremental.Rd index 85dab5f..dc6b3ca 100644 --- a/man/cleanIncremental.Rd +++ b/man/cleanIncremental.Rd @@ -29,7 +29,7 @@ cleanIncremental( } \seealso{ -Other Incremental: -\code{\link{cleanNonIncremental}()} +Other Incremental: +\code{\link[=cleanNonIncremental]{cleanNonIncremental()}} } \concept{Incremental} diff --git a/man/cleanNonIncremental.Rd b/man/cleanNonIncremental.Rd index a254bcf..ba78e68 100644 --- a/man/cleanNonIncremental.Rd +++ b/man/cleanNonIncremental.Rd @@ -24,7 +24,7 @@ cleanNonIncremental(file.path(tempdir(), 'incremental')) } \seealso{ -Other Incremental: -\code{\link{cleanIncremental}()} +Other Incremental: +\code{\link[=cleanIncremental]{cleanIncremental()}} } \concept{Incremental} diff --git a/man/createCaseSeriesSettings.Rd b/man/createCaseSeriesSettings.Rd index 0ed84af..8a546e3 100644 --- a/man/createCaseSeriesSettings.Rd +++ b/man/createCaseSeriesSettings.Rd @@ -69,8 +69,8 @@ caseSeriesSetting <- createCaseSeriesSettings( } \seealso{ -Other Aggregate: -\code{\link{createRiskFactorSettings}()}, -\code{\link{createTargetBaselineSettings}()} +Other Aggregate: +\code{\link[=createRiskFactorSettings]{createRiskFactorSettings()}}, +\code{\link[=createTargetBaselineSettings]{createTargetBaselineSettings()}} } \concept{Aggregate} diff --git a/man/createCharacterizationSettings.Rd b/man/createCharacterizationSettings.Rd index 7e1dc77..d37c407 100644 --- a/man/createCharacterizationSettings.Rd +++ b/man/createCharacterizationSettings.Rd @@ -9,8 +9,7 @@ createCharacterizationSettings( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL, - restrictWashoutToObs = TRUE + caseSeriesSettings = NULL ) } \arguments{ @@ -23,8 +22,6 @@ createCharacterizationSettings( \item{riskFactorSettings}{A list of riskFactorSettings settings} \item{caseSeriesSettings}{A list of caseSeriesSettings settings} - -\item{restrictWashoutToObs}{Whether the outcome washout can use outcomes outside the observation period when calling risk factor analysis} } \value{ Returns the connection to the sqlite database @@ -53,9 +50,9 @@ cSet <- createCharacterizationSettings( } \seealso{ -Other LargeScale: -\code{\link{loadCharacterizationSettings}()}, -\code{\link{runCharacterizationAnalyses}()}, -\code{\link{saveCharacterizationSettings}()} +Other LargeScale: +\code{\link[=loadCharacterizationSettings]{loadCharacterizationSettings()}}, +\code{\link[=runCharacterizationAnalyses]{runCharacterizationAnalyses()}}, +\code{\link[=saveCharacterizationSettings]{saveCharacterizationSettings()}} } \concept{LargeScale} diff --git a/man/createCharacterizationTables.Rd b/man/createCharacterizationTables.Rd index e412d4e..dc9e6ca 100644 --- a/man/createCharacterizationTables.Rd +++ b/man/createCharacterizationTables.Rd @@ -52,8 +52,8 @@ createCharacterizationTables( } \seealso{ -Other Database: -\code{\link{createSqliteDatabase}()}, -\code{\link{insertResultsToDatabase}()} +Other Database: +\code{\link[=createSqliteDatabase]{createSqliteDatabase()}}, +\code{\link[=insertResultsToDatabase]{insertResultsToDatabase()}} } \concept{Database} diff --git a/man/createDuringCovariateSettings.Rd b/man/createDuringCovariateSettings.Rd index 99c7254..1dffe35 100644 --- a/man/createDuringCovariateSettings.Rd +++ b/man/createDuringCovariateSettings.Rd @@ -105,7 +105,7 @@ settings <- createDuringCovariateSettings( } \seealso{ -Other CovariateSetting: -\code{\link{getDbDuringCovariateData}()} +Other CovariateSetting: +\code{\link[=getDbDuringCovariateData]{getDbDuringCovariateData()}} } \concept{CovariateSetting} diff --git a/man/createRiskFactorSettings.Rd b/man/createRiskFactorSettings.Rd index ef4ed33..66f6c4c 100644 --- a/man/createRiskFactorSettings.Rd +++ b/man/createRiskFactorSettings.Rd @@ -72,8 +72,8 @@ riskFactorSetting <- createRiskFactorSettings( } \seealso{ -Other Aggregate: -\code{\link{createCaseSeriesSettings}()}, -\code{\link{createTargetBaselineSettings}()} +Other Aggregate: +\code{\link[=createCaseSeriesSettings]{createCaseSeriesSettings()}}, +\code{\link[=createTargetBaselineSettings]{createTargetBaselineSettings()}} } \concept{Aggregate} diff --git a/man/createSqliteDatabase.Rd b/man/createSqliteDatabase.Rd index 8852ba4..a89dbf5 100644 --- a/man/createSqliteDatabase.Rd +++ b/man/createSqliteDatabase.Rd @@ -25,8 +25,8 @@ charResultDbCD <- createSqliteDatabase() } \seealso{ -Other Database: -\code{\link{createCharacterizationTables}()}, -\code{\link{insertResultsToDatabase}()} +Other Database: +\code{\link[=createCharacterizationTables]{createCharacterizationTables()}}, +\code{\link[=insertResultsToDatabase]{insertResultsToDatabase()}} } \concept{Database} diff --git a/man/createTargetBaselineSettings.Rd b/man/createTargetBaselineSettings.Rd index 18cfbc4..afc3522 100644 --- a/man/createTargetBaselineSettings.Rd +++ b/man/createTargetBaselineSettings.Rd @@ -46,8 +46,8 @@ aggregateSetting <- createTargetBaselineSettings( } \seealso{ -Other Aggregate: -\code{\link{createCaseSeriesSettings}()}, -\code{\link{createRiskFactorSettings}()} +Other Aggregate: +\code{\link[=createCaseSeriesSettings]{createCaseSeriesSettings()}}, +\code{\link[=createRiskFactorSettings]{createRiskFactorSettings()}} } \concept{Aggregate} diff --git a/man/getDbDuringCovariateData.Rd b/man/getDbDuringCovariateData.Rd index f421791..b6e5731 100644 --- a/man/getDbDuringCovariateData.Rd +++ b/man/getDbDuringCovariateData.Rd @@ -106,7 +106,7 @@ DatabaseConnector::disconnect(connection) } \seealso{ -Other CovariateSetting: -\code{\link{createDuringCovariateSettings}()} +Other CovariateSetting: +\code{\link[=createDuringCovariateSettings]{createDuringCovariateSettings()}} } \concept{CovariateSetting} diff --git a/man/insertResultsToDatabase.Rd b/man/insertResultsToDatabase.Rd index 37ba802..29179d8 100644 --- a/man/insertResultsToDatabase.Rd +++ b/man/insertResultsToDatabase.Rd @@ -84,8 +84,8 @@ Calls ResultModelManager uploadResults function to upload the csv files } \seealso{ -Other Database: -\code{\link{createCharacterizationTables}()}, -\code{\link{createSqliteDatabase}()} +Other Database: +\code{\link[=createCharacterizationTables]{createCharacterizationTables()}}, +\code{\link[=createSqliteDatabase]{createSqliteDatabase()}} } \concept{Database} diff --git a/man/loadCharacterizationSettings.Rd b/man/loadCharacterizationSettings.Rd index 096a482..b41ae6a 100644 --- a/man/loadCharacterizationSettings.Rd +++ b/man/loadCharacterizationSettings.Rd @@ -46,9 +46,9 @@ setting <- loadCharacterizationSettings(setPath) } \seealso{ -Other LargeScale: -\code{\link{createCharacterizationSettings}()}, -\code{\link{runCharacterizationAnalyses}()}, -\code{\link{saveCharacterizationSettings}()} +Other LargeScale: +\code{\link[=createCharacterizationSettings]{createCharacterizationSettings()}}, +\code{\link[=runCharacterizationAnalyses]{runCharacterizationAnalyses()}}, +\code{\link[=saveCharacterizationSettings]{saveCharacterizationSettings()}} } \concept{LargeScale} diff --git a/man/runCharacterizationAnalyses.Rd b/man/runCharacterizationAnalyses.Rd index 3b7966f..0014d86 100644 --- a/man/runCharacterizationAnalyses.Rd +++ b/man/runCharacterizationAnalyses.Rd @@ -139,9 +139,9 @@ runCharacterizationAnalyses( } \seealso{ -Other LargeScale: -\code{\link{createCharacterizationSettings}()}, -\code{\link{loadCharacterizationSettings}()}, -\code{\link{saveCharacterizationSettings}()} +Other LargeScale: +\code{\link[=createCharacterizationSettings]{createCharacterizationSettings()}}, +\code{\link[=loadCharacterizationSettings]{loadCharacterizationSettings()}}, +\code{\link[=saveCharacterizationSettings]{saveCharacterizationSettings()}} } \concept{LargeScale} diff --git a/man/saveCharacterizationSettings.Rd b/man/saveCharacterizationSettings.Rd index 1017994..55e5735 100644 --- a/man/saveCharacterizationSettings.Rd +++ b/man/saveCharacterizationSettings.Rd @@ -41,9 +41,9 @@ saveCharacterizationSettings( } \seealso{ -Other LargeScale: -\code{\link{createCharacterizationSettings}()}, -\code{\link{loadCharacterizationSettings}()}, -\code{\link{runCharacterizationAnalyses}()} +Other LargeScale: +\code{\link[=createCharacterizationSettings]{createCharacterizationSettings()}}, +\code{\link[=loadCharacterizationSettings]{loadCharacterizationSettings()}}, +\code{\link[=runCharacterizationAnalyses]{runCharacterizationAnalyses()}} } \concept{LargeScale} diff --git a/tests/testthat/test-CaseSeries.R b/tests/testthat/test-CaseSeries.R index c600b09..b8fbba1 100644 --- a/tests/testthat/test-CaseSeries.R +++ b/tests/testthat/test-CaseSeries.R @@ -457,7 +457,7 @@ data1 <- FeatureExtraction::getDbCovariateData( cohortTable = 'char_cohort_set1_db1', cohortDatabaseSchema = "main", cohortIds = c(10,20,40), # the targets - rowIdField = 'row_number', + rowIdField = 'row_id', exportToTable = FALSE, aggregated = TRUE, minCharacterizationMean = 0.01, @@ -483,7 +483,7 @@ data2 <- FeatureExtraction::getDbCovariateData( cohortTable = 'char_cohort_set1_db1', cohortDatabaseSchema = "main", cohortIds = c(10,20,40), # the targets - rowIdField = 'row_number', + rowIdField = 'row_id', exportToTable = FALSE, aggregated = TRUE, minCharacterizationMean = 0.01, @@ -506,7 +506,7 @@ data3 <- FeatureExtraction::getDbCovariateData( cohortTable = 'char_cohort_set1_db1', cohortDatabaseSchema = "main", cohortIds = c(10,20,40), # the targets - rowIdField = 'row_number', + rowIdField = 'row_id', exportToTable = FALSE, aggregated = TRUE, minCharacterizationMean = 0.01, diff --git a/tests/testthat/test-CohortGeneration.R b/tests/testthat/test-CohortGeneration.R index 53f4e19..a154f22 100644 --- a/tests/testthat/test-CohortGeneration.R +++ b/tests/testthat/test-CohortGeneration.R @@ -1,6 +1,122 @@ context("CohortGeneration") +test_that("generateOutcomeEras executes on sqlite", { + skipIfCreateTargetCohortSqlUnavailable() + + sqlitePath <- tempfile(fileext = ".sqlite") + on.exit(unlink(sqlitePath, force = TRUE), add = TRUE) + + connectionDetails <- DatabaseConnector::createConnectionDetails( + dbms = "sqlite", + server = sqlitePath + ) + connection <- DatabaseConnector::connect(connectionDetails = connectionDetails) + on.exit(DatabaseConnector::disconnect(connection), add = TRUE) + + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = "main", + tableName = "cohort", + data = data.frame( + cohort_definition_id = c(3, 3, 3, 3), + subject_id = c(1, 1, 1, 2), + cohort_start_date = as.Date(c("2020-01-01", "2020-06-01", "2021-07-01", "2020-03-01")), + cohort_end_date = as.Date(c("2020-01-10", "2020-06-10", "2021-07-10", "2020-03-05")) + ) + ) + + DatabaseConnector::executeSql( + connection = connection, + sql = "CREATE TABLE main.outcome_era (cohort_definition_id BIGINT, outcome_washout BIGINT, subject_id BIGINT, cohort_start_date DATE, cohort_end_date DATE);", + progressBar = FALSE, + reportOverallTime = FALSE + ) + + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "OutcomeEras.sql", + packageName = "Characterization", + dbms = "sqlite", + tempEmulationSchema = "main", + characterization_schema = "main", + outcome_era_table = "outcome_era", + outcome_ids = "3", + outcome_washout = 365, + cohort_schema = "main", + cohort_table = "cohort" + ) + + # let it run without stopping as test is later + testthat::expect_error( + DatabaseConnector::executeSql( + connection = connection, + sql = sql, + progressBar = FALSE, + reportOverallTime = FALSE + ), + NA + ) + + eras <- DatabaseConnector::querySql( + connection = connection, + sql = "SELECT cohort_definition_id, outcome_washout, subject_id, cohort_start_date, cohort_end_date FROM main.outcome_era ORDER BY subject_id, cohort_start_date" + ) + + expected <- data.frame( + cohort_definition_id = c(3, 3,3), + outcome_washout = c(365, 365, 365), + subject_id = c(1, 1, 2), + cohort_start_date = as.Date(c("2020-01-01", "2021-07-01","2020-03-01")), + cohort_end_date = as.Date(c("2020-06-10", "2021-07-10","2020-03-05")) + ) + + testthat::expect_equal(eras, expected) + + + # now test rerunning with another washout + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = "OutcomeEras.sql", + packageName = "Characterization", + dbms = "sqlite", + tempEmulationSchema = "main", + characterization_schema = "main", + outcome_era_table = "outcome_era", + outcome_ids = "3", + outcome_washout = 365*10, + cohort_schema = "main", + cohort_table = "cohort" + ) + + # let it run without stopping as test is later + testthat::expect_error( + DatabaseConnector::executeSql( + connection = connection, + sql = sql, + progressBar = FALSE, + reportOverallTime = FALSE + ), + NA + ) + + eras <- DatabaseConnector::querySql( + connection = connection, + sql = "SELECT cohort_definition_id, outcome_washout, subject_id, cohort_start_date, cohort_end_date FROM main.outcome_era ORDER BY subject_id, cohort_start_date" + ) + + expected <- data.frame( + cohort_definition_id = c(3,3,3,3,3), + outcome_washout = c(365, 3650, 365, 365,3650), + subject_id = c(1, 1, 1, 2, 2), + cohort_start_date = as.Date(c("2020-01-01","2020-01-01", "2021-07-01","2020-03-01", "2020-03-01")), + cohort_end_date = as.Date(c("2020-06-10","2021-07-10", "2021-07-10","2020-03-05", "2020-03-05")) + ) + + testthat::expect_equal(eras, expected) + + +}) + + test_that("getCohortJobs", { targetIds <- c(1, 2, 4) outcomeIds <- c(3) @@ -167,7 +283,7 @@ jobs <- getCohortJobs( testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 12) +testthat::expect_true(nrow(jobs$jobs) == (12 + 1)) # one more due to outcome eras testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, @@ -176,7 +292,7 @@ testthat::expect_true(sum(unique( ) == 3) -jobs <- Characterization:::getCohortJobs( +jobs <- getCohortJobs( characterizationSettings = characterizationSettings, mode = 'PatientLevelPrediction', nTargetJobs = 4 @@ -184,7 +300,7 @@ jobs <- Characterization:::getCohortJobs( testthat::expect_true(nrow(jobs$targets) == 6) testthat::expect_true(nrow(jobs$cases) == 3) -testthat::expect_true(nrow(jobs$jobs) == 12) +testthat::expect_true(nrow(jobs$jobs) == (12 + 1)) # 1 extra outcome era testthat::expect_true(sum(unique( c(ParallelLogger::convertJsonToSettings(jobs$jobs$settings[1])$targetIds, From ea5cd70e8c14e1145e894738cbe970a98e8c5c93 Mon Sep 17 00:00:00 2001 From: jreps Date: Mon, 13 Jul 2026 09:20:40 -0400 Subject: [PATCH 34/39] moving outcome era to before non case --- DESCRIPTION | 2 +- R/CohortGeneration.R | 42 ++++++++++++++------------- inst/sql/sql_server/TargetCohorts.sql | 2 +- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4441daf..f6c1d9b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: Characterization Type: Package Title: Implement Descriptive Studies Using the Common Data Model Version: 4.0.0 -Date: 2026-6-15 +Date: 2026-7-15 Authors@R: c( person("Jenna", "Reps", , "jreps@its.jnj.com", role = c("aut", "cre")), person("Patrick", "Ryan", , "ryan@ohdsi.org", role = c("aut")), diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index cd7a907..f93f712 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -402,6 +402,28 @@ getCohortJobs <- function( } + # add in job for outcome eras per washout + # only run if there are cases and mode is not Efficient + # since efficient mode doesnt need the outcomes + # THIS NEEDS TO BE RUN BEFOR NON-CASE generation + if(!is.null(nrow(cases))){ + if(mode != 'Efficient'){ + ooi <- unique(cases[, c('outcomeId', 'outcomeWashoutDays')]) + for(outcomeWashoutDay in unique(ooi$outcomeWashoutDays)){ + jobs <- rbind(jobs, data.frame( + functionName = 'generateOutcomeEras', + settings = as.character(ParallelLogger::convertSettingsToJson(list( + outcomeIds = ooi$outcomeId[ooi$outcomeWashoutDays == outcomeWashoutDay], + outcomeWashoutDays = outcomeWashoutDay + ) + )), + jobId = paste("outcome_eras",i,outcomeWashoutDay, sep = "_") + )) + } + } + } + + if(!is.null(nrow(cases))){ cases <- unique(cases) %>% @@ -509,26 +531,6 @@ getCohortJobs <- function( } - # add in job for outcome eras per washout - # only run if there are cases and mode is not Efficient - # since efficient mode doesnt need the outcomes - if(!is.null(nrow(cases))){ - if(mode != 'Efficient'){ - ooi <- unique(cases[, c('outcomeId', 'outcomeWashoutDays')]) - for(outcomeWashoutDay in unique(ooi$outcomeWashoutDays)){ - jobs <- rbind(jobs, data.frame( - functionName = 'generateOutcomeEras', - settings = as.character(ParallelLogger::convertSettingsToJson(list( - outcomeIds = ooi$outcomeId[ooi$outcomeWashoutDays == outcomeWashoutDay], - outcomeWashoutDays = outcomeWashoutDay - ) - )), - jobId = paste("outcome_eras",i,outcomeWashoutDay, sep = "_") - )) - } - } - } - # removing nTargetJobs if(!is.null(nrow(targets))){ targets <- targets %>% dplyr::select(-"nTargetJobs") diff --git a/inst/sql/sql_server/TargetCohorts.sql b/inst/sql/sql_server/TargetCohorts.sql index a814906..a8f56d3 100644 --- a/inst/sql/sql_server/TargetCohorts.sql +++ b/inst/sql/sql_server/TargetCohorts.sql @@ -130,7 +130,7 @@ subject_id, cohort_start_date, -- edit the end date if after study end {@study_end != ''}?{ -CASE WHEN @study_end < cohort_end_date THEN @study_end ELSE cohort_end_date END as cohort_end_date, +CASE WHEN CAST('@study_end' AS DATE) < cohort_end_date THEN CAST('@study_end' AS DATE) ELSE cohort_end_date END as cohort_end_date, } : {cohort_end_date,} observation_period_start_date, From c8ef4ed9799677979c05ed5ed4d9b78d6938f017 Mon Sep 17 00:00:00 2001 From: jreps Date: Tue, 28 Jul 2026 13:38:04 -0400 Subject: [PATCH 35/39] editing when minCharacterizationMean is called - added test for outcomeWashoutDays to make sure it is only a single value - edited risk factors/case series so that the minCharacterizationMean is applied after processing (otherwise it can make SMD seem bigger) --- R/CaseSeries.R | 16 +++++++++++----- R/RiskFactorAnalysis.R | 12 +++++++++--- .../sql_server/CaseSeriesBinaryExtraction.sql | 6 ++++++ .../sql_server/RiskFactorBinaryExtraction.sql | 3 +++ man/createCaseSeriesSettings.Rd | 6 +++--- man/createRiskFactorSettings.Rd | 2 +- 6 files changed, 33 insertions(+), 12 deletions(-) diff --git a/R/CaseSeries.R b/R/CaseSeries.R index 6208ce6..1b6e6b2 100644 --- a/R/CaseSeries.R +++ b/R/CaseSeries.R @@ -18,11 +18,11 @@ #' #' @param studyPopulationSettings A List of object created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts -#' @param outcomeWashoutDays Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis +#' @param outcomeWashoutDays A single integer value. Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis #' @template timeAtRisk #' @param caseCovariateSettings An object created using \code{createDuringCovariateSettings} -#' @param casePreTargetDuration The number of days prior to case index we use for FeatureExtraction -#' @param casePostOutcomeDuration The number of days prior to case index we use for FeatureExtraction +#' @param casePreTargetDuration A single integer value. The number of days prior to case index we use for FeatureExtraction +#' @param casePostOutcomeDuration A single integer value. The number of days prior to case index we use for FeatureExtraction #' @family Aggregate #' @return #' A list with the settings @@ -80,6 +80,11 @@ createCaseSeriesSettings <- function( errorMessages = errorMessages ) + # check outcomeWashoutDays is length 1 + if (length(outcomeWashoutDays) > 1) { + stop("Please add one outcomeWashoutDays per setting") + } + # check TAR - EFF edit if (length(riskWindowStart) > 1) { stop("Please add one time-at-risk per setting") @@ -237,7 +242,7 @@ computeCaseSeriesAnalyses <- function( rowIdField = 'row_id', covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettings), aggregated = TRUE, - minCharacterizationMean = minCharacterizationMean, + minCharacterizationMean = 0, #minCharacterizationMean, exportToTable = TRUE, targetDatabaseSchema = NULL, @@ -270,7 +275,8 @@ computeCaseSeriesAnalyses <- function( caseIds$characterizationCaseId*10+4, caseIds$characterizationCaseId*10+5), collapse = ','), - min_count = minCovariateCount + min_count = minCovariateCount, + min_characterization_mean = minCharacterizationMean ) tryCatch( diff --git a/R/RiskFactorAnalysis.R b/R/RiskFactorAnalysis.R index 82ea289..1004102 100644 --- a/R/RiskFactorAnalysis.R +++ b/R/RiskFactorAnalysis.R @@ -18,7 +18,7 @@ #' #' @param studyPopulationSettings A list of objects created using \code{createStudyPopulationSettings} that specifies target cohorts and inclusion criteria #' @param outcomeIds A list of cohortIds for the outcome cohorts -#' @param outcomeWashoutDays Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis +#' @param outcomeWashoutDays A single integer value. Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis #' @template timeAtRisk #' @param covariateSettings An object created using \code{FeatureExtraction::createCovariateSettings} #' @@ -96,6 +96,11 @@ createRiskFactorSettings <- function( errorMessages = errorMessages ) + # check outcomeWashoutDays is length 1 + if (length(outcomeWashoutDays) > 1) { + stop("Please add one outcomeWashoutDays per setting") + } + # check TAR - EFF edit if (length(riskWindowStart) > 1) { stop("Please add one time-at-risk per setting") @@ -262,7 +267,7 @@ computeRiskFactorAnalyses <- function( rowIdField = 'row_id', covariateSettings = ParallelLogger::convertJsonToSettings(settings$covariateSettings), aggregated = TRUE, - minCharacterizationMean = minCharacterizationMean, + minCharacterizationMean = 0, #minCharacterizationMean, exportToTable = TRUE, targetDatabaseSchema = NULL, @@ -298,7 +303,8 @@ computeRiskFactorAnalyses <- function( characterization_fe_table = '#fe_covariate_rf', efficient_mode = mode == 'Efficient', smd_min = minSMD, - min_count = minCovariateCount + min_count = minCovariateCount, + min_characterization_mean = minCharacterizationMean ) result <- Andromeda::andromeda() diff --git a/inst/sql/sql_server/CaseSeriesBinaryExtraction.sql b/inst/sql/sql_server/CaseSeriesBinaryExtraction.sql index a70b356..357b11b 100644 --- a/inst/sql/sql_server/CaseSeriesBinaryExtraction.sql +++ b/inst/sql/sql_server/CaseSeriesBinaryExtraction.sql @@ -52,5 +52,11 @@ covariate_id ) main_table WHERE (before_sum_value + during_sum_value + after_sum_value) >= @min_count +AND (ISNULL(before_average_value, 0) >= @min_characterization_mean + OR + ISNULL(during_average_value, 0) >= @min_characterization_mean + OR + ISNULL(after_average_value, 0) >= @min_characterization_mean + ) ; diff --git a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql index 62b56b4..1b4b98c 100644 --- a/inst/sql/sql_server/RiskFactorBinaryExtraction.sql +++ b/inst/sql/sql_server/RiskFactorBinaryExtraction.sql @@ -99,5 +99,8 @@ AND non_cases.covariate_id = cases.covariate_id WHERE abs(CASE WHEN st_dev = 0 THEN mean_diff ELSE mean_diff/st_dev END) >= @smd_min AND (ISNULL(non_case_sum_value, 0) + ISNULL(case_sum_value, 0) ) >= @min_count +AND (ISNULL(non_case_average_value, 0) >= @min_characterization_mean + OR + ISNULL(case_average_value, 0) >= @min_characterization_mean ) ; diff --git a/man/createCaseSeriesSettings.Rd b/man/createCaseSeriesSettings.Rd index 8a546e3..1ddea3c 100644 --- a/man/createCaseSeriesSettings.Rd +++ b/man/createCaseSeriesSettings.Rd @@ -25,7 +25,7 @@ createCaseSeriesSettings( \item{outcomeIds}{A list of cohortIds for the outcome cohorts} -\item{outcomeWashoutDays}{Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} +\item{outcomeWashoutDays}{A single integer value. Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} \item{riskWindowStart}{The start of the risk window (in days) relative to the `startAnchor`.} @@ -39,9 +39,9 @@ or `"cohort end"`.} \item{caseCovariateSettings}{An object created using \code{createDuringCovariateSettings}} -\item{casePreTargetDuration}{The number of days prior to case index we use for FeatureExtraction} +\item{casePreTargetDuration}{A single integer value. The number of days prior to case index we use for FeatureExtraction} -\item{casePostOutcomeDuration}{The number of days prior to case index we use for FeatureExtraction} +\item{casePostOutcomeDuration}{A single integer value. The number of days prior to case index we use for FeatureExtraction} } \value{ A list with the settings diff --git a/man/createRiskFactorSettings.Rd b/man/createRiskFactorSettings.Rd index 66f6c4c..1ca8322 100644 --- a/man/createRiskFactorSettings.Rd +++ b/man/createRiskFactorSettings.Rd @@ -34,7 +34,7 @@ createRiskFactorSettings( \item{outcomeIds}{A list of cohortIds for the outcome cohorts} -\item{outcomeWashoutDays}{Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} +\item{outcomeWashoutDays}{A single integer value. Patients with the outcome within outcomeWashout days prior to index are excluded from the risk factor analysis} \item{riskWindowStart}{The start of the risk window (in days) relative to the `startAnchor`.} From b6fa20ae4d239ae265ab24fee3df415a220525a4 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 31 Jul 2026 09:58:11 -0400 Subject: [PATCH 36/39] preparing to merge into develop preparing to merge into develop --- NEWS.md | 12 ++++++++++++ README.md | 31 +++++++++++++++++++++---------- vignettes/UsingPackage.Rmd | 2 +- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9ff17ea..60031d6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,15 @@ +Characterization 4.0.0 +====================== +- [enhancement] replaced targetId inputs to the settings with studyPopulationSettings that lets you specify + min prior observation, first in n days, age, date, gender and nesting cohort restrictions. +- [enhancement] outcome washout in risk factor is now used to combine outcome cohort entries that are within outcome washout days +- [enhancement] improved attrition capture: can now see fill attrition for study population +- [enhancement] changed csv file output to split up attrition into case_attrition and target_attrition, added + case_counts and target_counts and added time_to_event_settings/dechallenge_rechallenge_settings that + let user quickly see what study population and outcome pairs were included. +- [bug fix] fixed issues when dividing by zero in SMD calculation +- [bug fix] replaced IFNULL with ISNULL + Characterization 3.0.1 ====================== - Fix issue with uploading results into database for shiny viewer (spacing was added to csv and causing issues and continuous covariates that are floats were incorrectly bigints) diff --git a/README.md b/README.md index 6c9527e..de05e6f 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,12 @@ connectionDetails <- Characterization::exampleOmopConnectionDetails() targetIds <- c(1,2,4) outcomeIds <- c(3) - timeToEventSettings1 <- createTimeToEventSettings( - targetIds = 1, - outcomeIds = c(3,4) - ) - timeToEventSettings2 <- createTimeToEventSettings( - targetIds = 2, + timeToEventSettings <- createTimeToEventSettings( + studyPopulationSettings = createStudyPopulationSettings( + targetIds = c(1,2), + limitToFirstInNDays = 0, + minPriorObservation = 0 + ), outcomeIds = c(3,4) ) @@ -55,7 +55,11 @@ targetIds <- c(1,2,4) ) riskFactorSettings1 <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 99999, # first exposure + minPriorObservation = 365 # requiring 365 days prior obs + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = 'cohort start', @@ -69,7 +73,11 @@ targetIds <- c(1,2,4) ) riskFactorSettings2 <- createRiskFactorSettings( - targetIds = targetIds, + studyPopulationSettings = createStudyPopulationSettings( + targetIds = targetIds, + limitToFirstInNDays = 99999, # first exposure + minPriorObservation = 365 # requiring 365 days prior obs + ), outcomeIds = outcomeIds, riskWindowStart = 1, startAnchor = 'cohort start', @@ -82,8 +90,7 @@ targetIds <- c(1,2,4) characterizationSettings <- createCharacterizationSettings( timeToEventSettings = list( - timeToEventSettings1, - timeToEventSettings2 + timeToEventSettings ), dechallengeRechallengeSettings = list( dechallengeRechallengeSettings @@ -127,6 +134,10 @@ Installation 2. In R, use the following commands to download and install Characterization: ```r + # CRAN + install.packages('Characterization') + + # GitHub install.packages("remotes") remotes::install_github("ohdsi/Characterization") ``` diff --git a/vignettes/UsingPackage.Rmd b/vignettes/UsingPackage.Rmd index 1edf0fb..cf7d066 100644 --- a/vignettes/UsingPackage.Rmd +++ b/vignettes/UsingPackage.Rmd @@ -34,7 +34,7 @@ This vignette describes how you can use the Characterization package for various First we need to install the `Characterization` package: ```{r tidy=TRUE,eval=FALSE} -remotes::install_github("ohdsi/Characterization") +install.packages('Characterization') ``` and then load it: From 0a62bf71e89557df352516e2b10fcecaf3ff0b25 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 31 Jul 2026 10:47:27 -0400 Subject: [PATCH 37/39] removing restrictWashoutToObs - fixing merge conflict issue where restrictWashoutToObs got added --- R/CohortGeneration.R | 7 ++----- R/RunCharacterization.R | 7 ++----- man/createCharacterizationSettings.Rd | 5 +---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/R/CohortGeneration.R b/R/CohortGeneration.R index 6f26040..02ae543 100644 --- a/R/CohortGeneration.R +++ b/R/CohortGeneration.R @@ -204,7 +204,7 @@ generateCohorts <- function( settings = ParallelLogger::convertJsonToSettings(cohortJobs$jobs[i,"settings"]), jobId = cohortJobs$jobs[i, "jobId"], - + outcomeEraTable = outcomeEraTableWithHash ) @@ -744,7 +744,6 @@ generateNonCases <- function( jobId, mode, incremental, - restrictWashoutToObs, ... ){ @@ -780,9 +779,7 @@ generateNonCases <- function( cohort_table = outcomeTable, use_plp = mode == 'PatientLevelPrediction', - use_ci = mode == 'CohortIncidence', - - restrict_washout_to_obs = restrictWashoutToObs + use_ci = mode == 'CohortIncidence' ) DatabaseConnector::executeSql( diff --git a/R/RunCharacterization.R b/R/RunCharacterization.R index b90117b..7f293d3 100644 --- a/R/RunCharacterization.R +++ b/R/RunCharacterization.R @@ -10,7 +10,6 @@ #' @param targetBaselineSettings A list of targetBaselineSettings settings #' @param riskFactorSettings A list of riskFactorSettings settings #' @param caseSeriesSettings A list of caseSeriesSettings settings -#' @param restrictWashoutToObs Whether to restrict to outcomes in observation period for washout in risk factors #' @family LargeScale #' #' @return @@ -38,8 +37,7 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL, - restrictWashoutToObs = TRUE + caseSeriesSettings = NULL ) { errorMessages <- checkmate::makeAssertCollection() @@ -92,8 +90,7 @@ createCharacterizationSettings <- function( dechallengeRechallengeSettings = dechallengeRechallengeSettings, targetBaselineSettings = targetBaselineSettings, riskFactorSettings = riskFactorSettings, - caseSeriesSettings = caseSeriesSettings, - restrictWashoutToObs = restrictWashoutToObs + caseSeriesSettings = caseSeriesSettings ) # update the settings replace the popSet with the characterizationTargetIds diff --git a/man/createCharacterizationSettings.Rd b/man/createCharacterizationSettings.Rd index 79ec1eb..d37c407 100644 --- a/man/createCharacterizationSettings.Rd +++ b/man/createCharacterizationSettings.Rd @@ -9,8 +9,7 @@ createCharacterizationSettings( dechallengeRechallengeSettings = NULL, targetBaselineSettings = NULL, riskFactorSettings = NULL, - caseSeriesSettings = NULL, - restrictWashoutToObs = TRUE + caseSeriesSettings = NULL ) } \arguments{ @@ -23,8 +22,6 @@ createCharacterizationSettings( \item{riskFactorSettings}{A list of riskFactorSettings settings} \item{caseSeriesSettings}{A list of caseSeriesSettings settings} - -\item{restrictWashoutToObs}{Whether to restrict to outcomes in observation period for washout in risk factors} } \value{ Returns the connection to the sqlite database From 71cd59e1cade6200166b3e69aae01c2bf790a3aa Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 31 Jul 2026 13:30:22 -0400 Subject: [PATCH 38/39] shiny updates - updated required OhdsiShinyModules for shiny app - added missing columns to database_meta_data when viewing shiny to prevent annoying error message. --- R/ViewShiny.R | 13 ++++++++++++- inst/shinyConfigUpdate.json | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/R/ViewShiny.R b/R/ViewShiny.R index 58c94ef..d570635 100644 --- a/R/ViewShiny.R +++ b/R/ViewShiny.R @@ -141,7 +141,18 @@ prepareCharacterizationShiny <- function( tableName = "DATABASE_META_DATA", data = data.frame( databaseId = dbIds, - cdmSourceAbbreviation = paste0("database ", dbIds) + cdmSourceName = paste0("database ", dbIds), + cdmSourceAbbreviation = paste0("database ", dbIds), + cdmHolder = 'NA', + sourceDescription = 'NA', + sourceDocumentationReference = 'NA', + cdmEtlReference = 'NA', + sourceReleaseDate = 'NA', + cdmReleaseDate = 'NA', + cdmVersion = 'NA', + cdmVersionConceptId = 'NA', + vocabularyVersion = 'NA', + maxObsPeriodEndDate = 'NA' ), camelCaseToSnakeCase = TRUE ) diff --git a/inst/shinyConfigUpdate.json b/inst/shinyConfigUpdate.json index dc6965f..e643455 100644 --- a/inst/shinyConfigUpdate.json +++ b/inst/shinyConfigUpdate.json @@ -5,7 +5,7 @@ "tabName": "About", "tabText": "About", "shinyModulePackage": "OhdsiShinyModules", - "shinyModulePackageVersion": "2.2.0", + "shinyModulePackageVersion": "3.6.0", "installSource": "github", "gitHubRepo": "ohdsi", "uiFunction": "aboutViewer", @@ -19,7 +19,7 @@ "tabName": "Characterization", "tabText": "Characterization", "shinyModulePackage": "OhdsiShinyModules", - "shinyModulePackageVersion": "2.2.0", + "shinyModulePackageVersion": "3.6.0", "installSource": "github", "gitHubRepo": "ohdsi", "uiFunction": "characterizationViewer", From 8aa53f98d3935065b62c5874b9b3a21bd98a6945 Mon Sep 17 00:00:00 2001 From: jreps Date: Fri, 31 Jul 2026 14:36:42 -0400 Subject: [PATCH 39/39] fixing pkdown --- R/StudyPopulation.R | 2 +- man/createStudyPopulationSettings.Rd | 5 +++++ man/exampleOmopConnectionDetails.Rd | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/R/StudyPopulation.R b/R/StudyPopulation.R index 31ba619..a741256 100644 --- a/R/StudyPopulation.R +++ b/R/StudyPopulation.R @@ -11,7 +11,7 @@ #' @param studyStartDate The earliest date to be included into the target. Date format is 'yyyymmdd'. #' @param studyEndDate The latest date to be included into the target. Date format is 'yyyymmdd'. #' @param genderConceptIds A target cohort subject's gender concept to restrict to -#' +#' @family helper #' #' @return #' A data.frame containing all the settings required diff --git a/man/createStudyPopulationSettings.Rd b/man/createStudyPopulationSettings.Rd index 0688e41..f75194b 100644 --- a/man/createStudyPopulationSettings.Rd +++ b/man/createStudyPopulationSettings.Rd @@ -54,3 +54,8 @@ populationSettings <- createStudyPopulationSettings( minAge = 18 ) } +\seealso{ +Other helper: +\code{\link[=exampleOmopConnectionDetails]{exampleOmopConnectionDetails()}} +} +\concept{helper} diff --git a/man/exampleOmopConnectionDetails.Rd b/man/exampleOmopConnectionDetails.Rd index b09de2a..cf9287c 100644 --- a/man/exampleOmopConnectionDetails.Rd +++ b/man/exampleOmopConnectionDetails.Rd @@ -23,5 +23,9 @@ conDet <- exampleOmopConnectionDetails() connectionHandler <- ResultModelManager::ConnectionHandler$new(conDet) +} +\seealso{ +Other helper: +\code{\link[=createStudyPopulationSettings]{createStudyPopulationSettings()}} } \concept{helper}