diff --git a/.copyrightconfig b/.copyrightconfig new file mode 100644 index 000000000..ba242e11f --- /dev/null +++ b/.copyrightconfig @@ -0,0 +1,14 @@ +# COPYRIGHT VALIDATION CONFIG +# --------------------------------- +# Required start year (keep fixed; end year auto-updates in check output) +startyear: 2010 + +# Optional exclusions list (comma-separated). Leave commented if none. +# Rules: +# - Relative paths (no leading ./) +# - Simple * wildcard only (no recursive **) +# - Use sparingly (third_party, generated, binary assets) +# - Dotfiles already skipped automatically +# Enable by removing the leading '# ' from the next line and editing values. +# filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml diff --git a/.env b/.env index ab5dd6526..c67e5ae05 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ # Defines environment variables for docker-compose. # Can be overridden via e.g. `MARKLOGIC_TAG=latest-10.0 docker-compose up -d --build`. -MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest MARKLOGIC_LOGS_VOLUME=./docker/marklogic/logs +MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12 -# This image should be used instead of the above image when testing functions that only work with MarkLogic 12. -#MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12 +# Latest public release +#MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index f2a31ab99..d11ced4a0 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -1,4 +1,4 @@ -name: 🏷️ JIRA ID Validator +name: PR Workflow on: # Using pull_request_target instead of pull_request to handle PRs from forks @@ -14,3 +14,10 @@ jobs: with: # Pass the PR title from the event context pr-title: ${{ github.event.pull_request.title }} + copyright-validation: + name: © Validate Copyright Headers + uses: marklogic/pr-workflows/.github/workflows/copyright-check.yml@main + permissions: + contents: read + pull-requests: write + issues: write \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1bf7c14e2..d933dc310 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ ml-development-tools/src/test/java/com/marklogic/client/test/dbfunction/generate .vscode docker/ + +.kotlin + +dep.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9c2bb310..68b2f8c37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,11 +7,7 @@ To build the client locally, complete the following steps: 1. Clone this repository on your machine. 2. Choose the appropriate branch (usually develop) -3. Ensure you are using Java 8 or Java 11 or Java 17 (the JVM version used to compile should not matter as compiler flags -are set to ensure the compiled code will run on Java 8; Jenkins pipelines also exist to ensure that the tests pass on -Java 8, 11, and 17, and thus they should for you locally as well; note that if you load the project into an IDE, you -should use Java 8 in case your IDE does not process the build.gradle config that conditionally brings in JAXB dependencies -required by Java 9+.) +3. Ensure you are using Java 17. 4. Verify that you can build the client by running `./gradlew build -x test` "Running the tests" in the context of developing and submitting a pull request refers to running the tests found diff --git a/Jenkinsfile b/Jenkinsfile index 98e6d169d..080b15cf3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,31 +1,28 @@ @Library('shared-libraries') _ -def getJava(){ - if(env.JAVA_VERSION=="JAVA17"){ - return "/home/builder/java/jdk-17.0.2" - }else if(env.JAVA_VERSION=="JAVA11"){ - return "/home/builder/java/jdk-11.0.2" - }else if(env.JAVA_VERSION=="JAVA21"){ +def getJavaHomePath() { + if (env.JAVA_VERSION == "JAVA21") { return "/home/builder/java/jdk-21.0.1" - }else{ - return "/home/builder/java/openjdk-1.8.0-262" + } else { + return "/home/builder/java/jdk-17.0.2" } } -def setupDockerMarkLogic(String image){ +def setupDockerMarkLogic(String image) { cleanupDocker() - sh label:'mlsetup', script: '''#!/bin/bash + sh label: 'mlsetup', script: '''#!/bin/bash echo "Removing any running MarkLogic server and clean up MarkLogic data directory" sudo /usr/local/sbin/mladmin remove sudo /usr/local/sbin/mladmin cleandata cd java-client-api docker compose down -v || true docker volume prune -f - echo "Using image: "'''+image+''' - docker pull '''+image+''' - MARKLOGIC_IMAGE='''+image+''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build + echo "Using image: "''' + image + ''' + docker pull ''' + image + ''' + MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build echo "Waiting for MarkLogic server to initialize." sleep 60s + export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH ./gradlew mlTestConnections @@ -36,25 +33,31 @@ def setupDockerMarkLogic(String image){ def runTests(String image) { setupDockerMarkLogic(image) - sh label:'run marklogic-client-api tests', script: '''#!/bin/bash + sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - mkdir -p marklogic-client-api/build/test-results/test + + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup + rm -rf ~/.m2/repository/com/squareup/okhttp3/ + + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test + ./gradlew marklogic-client-api:test || true ''' - sh label:'run ml-development-tools tests', script: '''#!/bin/bash + sh label: 'run ml-development-tools tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - mkdir -p ml-development-tools/build/test-results/test ./gradlew ml-development-tools:test || true ''' - sh label:'run fragile functional tests', script: '''#!/bin/bash + sh label: 'run fragile functional tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH @@ -63,7 +66,7 @@ def runTests(String image) { ./gradlew marklogic-client-api-functionaltests:runFragileTests || true ''' - sh label:'run fast functional tests', script: '''#!/bin/bash + sh label: 'run fast functional tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH @@ -71,7 +74,7 @@ def runTests(String image) { ./gradlew marklogic-client-api-functionaltests:runFastFunctionalTests || true ''' - sh label:'run slow functional tests', script: '''#!/bin/bash + sh label: 'run slow functional tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH @@ -85,35 +88,52 @@ def runTests(String image) { def runTestsWithReverseProxy(String image) { setupDockerMarkLogic(image) - sh label:'run fragile functional tests with reverse proxy', script: '''#!/bin/bash + sh label: 'run marklogic-client-api tests with reverse proxy', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup + rm -rf ~/.m2/repository/com/squareup/okhttp3/ + + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test + + echo "Running marklogic-client-api tests with reverse proxy." + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test || true + ''' + + sh label: 'run fragile functional tests with reverse proxy', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true ''' - sh label:'run fast functional tests with reverse proxy', script: '''#!/bin/bash + sh label: 'run fast functional tests with reverse proxy', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true ''' - sh label:'run slow functional tests with reverse proxy', script: '''#!/bin/bash + sh label: 'run slow functional tests with reverse proxy', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true ''' postProcessTestResults() } def postProcessTestResults() { - sh label:'post-test-process', script: ''' + sh label: 'post-test-process', script: ''' cd java-client-api mkdir -p marklogic-client-api-functionaltests/build/test-results/runFragileTests mkdir -p marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests @@ -132,7 +152,7 @@ def postProcessTestResults() { } def tearDownDocker() { - sh label:'tearDownDocker', script: '''#!/bin/bash + sh label: 'tearDownDocker', script: '''#!/bin/bash cd java-client-api docker compose down -v || true docker volume prune -f @@ -140,62 +160,73 @@ def tearDownDocker() { cleanupDocker() } -pipeline{ - agent {label 'javaClientLinuxPool'} - - options { - checkoutToSubdirectory 'java-client-api' - buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10') - } - - parameters { - booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') - string(name: 'Email', defaultValue: '' ,description: 'Who should I say send the email to?') - string(name: 'JAVA_VERSION', defaultValue: 'JAVA8' ,description: 'Who should I say send the email to?') - } - - environment { - JAVA_HOME_DIR= getJava() - GRADLE_DIR =".gradle" - DMC_USER = credentials('MLBUILD_USER') - DMC_PASSWORD = credentials('MLBUILD_PASSWORD') - } - - stages { - stage('pull-request-tests') { - when { - not { - expression {return params.regressions} - } - } - steps { - setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:12.0.0-ubi-rootless-2.2.0") - sh label:'run marklogic-client-api tests', script: '''#!/bin/bash +pipeline { + agent { label 'javaClientLinuxPool' } + + options { + checkoutToSubdirectory 'java-client-api' + buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10') + } + + parameters { + booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') + string(name: 'JAVA_VERSION', defaultValue: 'JAVA17', description: 'Either JAVA17 or JAVA21') + } + + environment { + JAVA_HOME_DIR = getJavaHomePath() + GRADLE_DIR = ".gradle" + DMC_USER = credentials('MLBUILD_USER') + DMC_PASSWORD = credentials('MLBUILD_PASSWORD') + } + + stages { + + stage('pull-request-tests') { + when { + not { + expression { return params.regressions } + } + } + steps { + setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") + sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - ./gradlew cleanTest marklogic-client-api:test - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true + + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup + rm -rf ~/.m2/repository/com/squareup/okhttp3/ + + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test + + echo "Run a sufficient number of tests to verify the PR." + ./gradlew marklogic-client-api:test --tests ReadDocumentPageTest || true + + echo "Run a test with the reverse proxy server to ensure it's fine." + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:test --tests SearchWithPageLengthTest || true ''' - junit '**/build/**/TEST*.xml' - } + } post { always { + junit '**/build/**/TEST*.xml' updateWorkspacePermissions() tearDownDocker() } } - } - stage('publish'){ - when { - branch 'develop' - not { - expression {return params.regressions} - } - } - steps{ - sh label:'publish', script: '''#!/bin/bash + } + stage('publish') { + when { + branch 'develop' + not { + expression { return params.regressions } + } + } + steps { + sh label: 'publish', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH @@ -203,84 +234,65 @@ pipeline{ cd java-client-api ./gradlew publish ''' - } - } + } + } stage('regressions-11') { when { allOf { branch 'develop' - expression {return params.regressions} + expression { return params.regressions } } } steps { runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") - junit '**/build/**/TEST*.xml' } post { always { + junit '**/build/**/TEST*.xml' updateWorkspacePermissions() tearDownDocker() } } } - stage('regressions-11-reverseProxy') { - when { - allOf { - branch 'develop' - expression {return params.regressions} - } - } - steps { - runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") - junit '**/build/**/TEST*.xml' - } - post { - always { - updateWorkspacePermissions() - tearDownDocker() - } - } - } + // Latest run had 87 errors, which have been added to MLE-24523 for later research. +// stage('regressions-12-reverseProxy') { +// when { +// allOf { +// branch 'develop' +// expression {return params.regressions} +// } +// } +// steps { +// runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") +// } +// post { +// always { +// junit '**/build/**/TEST*.xml' +// updateWorkspacePermissions() +// tearDownDocker() +// } +// } +// } stage('regressions-12') { when { allOf { branch 'develop' - expression {return params.regressions} + expression { return params.regressions } } } steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:12.0.0-ubi-rootless-2.2.0") - junit '**/build/**/TEST*.xml' + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") } post { always { + junit '**/build/**/TEST*.xml' updateWorkspacePermissions() tearDownDocker() } } } - - stage('regressions-10.0') { - when { - allOf { - branch 'develop' - expression {return params.regressions} - } - } - steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10") - junit '**/build/**/TEST*.xml' - } - post { - always { - updateWorkspacePermissions() - tearDownDocker() - } - } - } - - } + } } diff --git a/NOTICE.txt b/NOTICE.txt index fe3ed8d43..700be2e15 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10,15 +10,14 @@ product and version for which you are requesting source code. Third Party Notices -jackson-databind 2.19.0 (Apache-2.0) -jackson-dataformat-csv 2.19.0 (Apache-2.0) -okhttp 4.12.0 (Apache-2.0) -logging-interceptor 4.12.0 (Apache-2.0) -jakarta.mail 2.0.1 (EPL-1.0) -okhttp-digest 2.7 (Apache-2.0) -jakarta.xml.bind-api 3.0.1 (EPL-1.0) -javax.ws.rs-api 2.1.1 (CDDL-1.1) -jaxb-runtime 3.0.2 (CDDL-1.1) +jackson-databind 2.20.0 (Apache-2.0) +jackson-dataformat-csv 2.20.0 (Apache-2.0) +okhttp 5.2.0 (Apache-2.0) +logging-interceptor 5.2.0 (Apache-2.0) +jakarta.mail 2.0.2 (EPL-1.0) +okhttp-digest 3.1.1 (Apache-2.0) +jakarta.xml.bind-api 4.0.4 (EPL-1.0) +jaxb-runtime 4.0.6 (CDDL-1.1) slf4j-api 2.0.17 (Apache-2.0) Common Licenses @@ -29,41 +28,37 @@ Eclipse Public License 1.0 (EPL-1.0) Third-Party Components -The following is a list of the third-party components used by the MarkLogic® for Java Client 7.2.0 (last updated July 21, 2025): +The following is a list of the third-party components used by the MarkLogic® for Java Client 8.0.0 (last updated October 29, 2025): -jackson-databind 2.19.0 (Apache-2.0) +jackson-databind 2.20.0 (Apache-2.0) https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -jackson-dataformat-csv 2.19.0 (Apache-2.0) +jackson-dataformat-csv 2.20.0 (Apache-2.0) https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -okhttp 4.12.0 (Apache-2.0) +okhttp 5.2.0 (Apache-2.0) https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -logging-interceptor 4.12.0 (Apache-2.0) +logging-interceptor 5.2.0 (Apache-2.0) https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -jakarta.mail 2.0.1 (Apache-2.0) +jakarta.mail 2.0.2 (Apache-2.0) https://repo1.maven.org/maven2/com/sun/mail/jakarta.mail/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -okhttp-digest 2.7 (Apache-2.0) +okhttp-digest 3.1.1 (Apache-2.0) https://repo1.maven.org/maven2/io/github/rburgst/okhttp-digest/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -jakarta.xml.bind-api 3.0.1 (Apache-2.0) +jakarta.xml.bind-api 4.0.4 (Apache-2.0) https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -javax.ws.rs-api 2.1.1 (Apache-2.0) -https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jaxb-runtime 3.0.2 (Apache-2.0) +jaxb-runtime 4.0.6 (Apache-2.0) https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) @@ -73,7 +68,7 @@ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) Common Licenses -The following is a list of the third-party components used by the MarkLogic® for Java Client 7.2.0 (last updated July 21, 2025): +The following is a list of the third-party components used by the MarkLogic® for Java Client 8.0.0 (last updated October 29, 2025): Apache License 2.0 (Apache-2.0) https://spdx.org/licenses/Apache-2.0.html diff --git a/README.md b/README.md index be9e1944e..7ba1a473c 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,13 @@ The client supports the following core features of the MarkLogic database: * Execute multi-statement transactions so changes to multiple documents succeed or fail together. * Call Data Services via a Java interface on the client for data functionality implemented by an endpoint on the server. -The client is tested on Java 8, 11, 17, and 21 and can safely be used on each of those major Java versions. The client -may work on more recent major versions of Java but has not been thoroughly tested on those yet. +## System Requirements -If you are using Java 11 or higher and intend to use [JAXB](https://docs.oracle.com/javase/tutorial/jaxb/intro/), please see the section below for ensuring that the -necessary dependencies are available in your application's classpath. +As of the 8.0.0 release, the Java Client requires Java 17 or Java 21. + +Prior releases are compatible with Java 8, 11, 17, and 21. + +For compatibility with MarkLogic server versions, please see the [Compatibility Matrix](https://developer.marklogic.com/products/support-matrix/#java-client-api). ## QuickStart @@ -33,13 +35,13 @@ To use the client in your [Maven](https://maven.apache.org/) project, include th com.marklogic marklogic-client-api - 7.2.0 + 8.0.0 To use the client in your [Gradle](https://gradle.org/) project, include the following in your `build.gradle` file: dependencies { - implementation "com.marklogic:marklogic-client-api:7.2.0" + implementation "com.marklogic:marklogic-client-api:8.0.0" } Next, read [The Java API in Five Minutes](http://developer.marklogic.com/try/java/index) to get started. diff --git a/build.gradle b/build.gradle index f5d11837e..6ad238fe4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,53 +1,59 @@ -// Copyright © 2024 MarkLogic Corporation. All Rights Reserved. - -// We need the properties plugin to work on both marklogic-client-api and test-app. The 'plugins' Gradle syntax can't be -// used for that. So we have to add the properties plugin to the buildscript classpath and then apply the properties -// plugin via subprojects below. -buildscript { - repositories { - maven { - url = "https://plugins.gradle.org/m2/" - } - } - dependencies { - classpath "net.saliman:gradle-properties-plugin:1.5.2" - } -} +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ subprojects { - apply plugin: "net.saliman.properties" - apply plugin: 'java' + apply plugin: 'java-library' - tasks.withType(JavaCompile) { - options.encoding = 'UTF-8' - options.compilerArgs += ["-Xlint:unchecked", "-Xlint:deprecation"] - } - - // To ensure that the Java Client continues to support Java 8, both source and target compatibility are set to 1.8. java { - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } } configurations { testImplementation.extendsFrom compileOnly + + all { + resolutionStrategy { + // Forcing the latest commons-lang3 version to eliminate CVEs. + force "org.apache.commons:commons-lang3:3.19.0" + } + } } repositories { mavenLocal() mavenCentral() + + // Needed so that ml-development-tools can resolve snapshots of marklogic-client-api. + maven { + url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" + } } - test { - systemProperty "file.encoding", "UTF-8" - systemProperty "javax.xml.stream.XMLOutputFactory", "com.sun.xml.internal.stream.XMLOutputFactoryImpl" + // Allows for identifying compiler warnings and treating them as errors. + tasks.withType(JavaCompile).configureEach { + options.compilerArgs += ["-Xlint:unchecked", "-Xlint:deprecation", "-Werror"] + options.deprecation = true + options.warnings = true } - // Until we do a cleanup of javadoc errors, the build (and specifically the javadoc task) fails on Java 11 - // and higher. Preventing that until the cleanup can occur. - javadoc.failOnError = false + tasks.withType(Test).configureEach { + // Can't use useJUnitPlatform here as it breaks ml-development-tools + testLogging { + events = ['started', 'passed', 'skipped', 'failed'] + exceptionFormat = 'full' + } + } - // Ignores warnings on param tags with no descriptions. Will remove this once javadoc errors are addressed. - // Until then, it's just a lot of noise. - javadoc.options.addStringOption('Xdoclint:none', '-quiet') + tasks.withType(Javadoc).configureEach { + // Until we do a cleanup of javadoc errors, the build (and specifically the javadoc task) fails on Java 11 + // and higher. Preventing that until the cleanup can occur. + failOnError = false + + // Ignores warnings on param tags with no descriptions. Will remove this once javadoc errors are addressed. + // Until then, it's just a lot of noise. + options.addStringOption('Xdoclint:none', '-quiet') + } } diff --git a/docker-compose.yaml b/docker-compose.yaml index 4a851837e..9d1dab27e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,14 +10,15 @@ services: - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=admin - MARKLOGIC_ADMIN_PASSWORD=admin + # The NET_RAW capability allows a process to create raw sockets. Polaris does not like that. + # This setting removes the NET_RAW capability from the container. + cap_drop: + - NET_RAW volumes: - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs ports: - "8000-8002:8000-8002" - - "8010-8014:8010-8014" - - "8022:8022" - - "8054-8059:8054-8059" - - "8093:8093" + - "8010-8015:8010-8015" # Range of ports used by app servers, at least one of which - 8015 - is created by a test. volumes: marklogicLogs: diff --git a/examples/build.gradle b/examples/build.gradle index ba3dfc93c..3e123799c 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -1,28 +1,27 @@ -// Copyright © 2025 MarkLogic Corporation. All Rights Reserved. - -plugins { - id "java-library" -} +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ dependencies { implementation project(':marklogic-client-api') + implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" // The 'api' configuration is used so that the test configuration in marklogic-client-api doesn't have to declare // all of these dependencies. This library project won't otherwise be depended on by anything else as it's not // setup for publishing. - api 'com.squareup.okhttp3:okhttp:4.12.0' - api 'io.github.rburgst:okhttp-digest:2.7' + api "com.squareup.okhttp3:okhttp:${okhttpVersion}" + api 'io.github.rburgst:okhttp-digest:3.1.1' api 'org.slf4j:slf4j-api:2.0.17' api "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" api 'org.jdom:jdom2:2.0.6.1' - api 'org.dom4j:dom4j:2.1.4' - api 'com.google.code.gson:gson:2.10.1' + api 'org.dom4j:dom4j:2.2.0' + api 'com.google.code.gson:gson:2.13.2' api 'net.sourceforge.htmlcleaner:htmlcleaner:2.29' - api ('com.opencsv:opencsv:5.11.2') { + api ('com.opencsv:opencsv:5.12.0') { // Excluding this due to a security vulnerability, and the test for the example that uses this library // passes without this on the classpath. exclude module: "commons-beanutils" } - api 'org.apache.commons:commons-lang3:3.18.0' + api 'org.apache.commons:commons-lang3:3.19.0' } diff --git a/examples/src/main/java/com/marklogic/client/example/cookbook/README.md b/examples/src/main/java/com/marklogic/client/example/cookbook/README.md deleted file mode 100644 index f47296fd9..000000000 --- a/examples/src/main/java/com/marklogic/client/example/cookbook/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Using Cookbook Examples - -The most important use of cookbook examples is reading the source code. You -can do this on [github](https://github.com/marklogic/java-client-api) or on -your machine once you've cloned the code from github. - -To run the examples, first edit the -[Example.properties](../../../../../../resources/Example.properties) file in the -distribution to specify the connection parameters for your server. Most -Cookbook examples have a main method, so they can be run from the command-line -like so: - - java -cp $CLASSPATH com.marklogic.client.example.cookbook.DocumentWrite - -This, of course, requires that you have all necessary dependencies in the env -variable $CLASSPATH. You can get the classpath for your machine by executing the the following gradle task - - ./gradlew printClasspath - -# Testing Cookbook Examples - -Most cookbook examples pass their unit test if they run without error. First -edit the [Example.properties](../../../../../../resources/Example.properties) file -in the distribution to specify the connection parameters for your server. Then -run `./gradlew test` while specifying the unit test you want to run, for example: - - ./gradlew java-client-api:test -Dtest.single=DocumentWriteTest - -The above command runs the DocumentWriteTest unit test in java-client-api sub project. - -# Creating a Cookbook Example - -We encourage community-contributed cookbook examples! Make sure you follow -the guidelines in [CONTRIBUTING.md](../../../../../../../../CONTRIBUTING.md) -when you submit a pull request. Each cookbook example should be runnable from -the command-line, so it should have a static `main` method. The approach in -the code should come as close as possible to production code (code one would -reasonably expect to use in a production application), while remaining as -simple as possible to facilitate grokking for newbies to the Java Client API. -It should have helpful comments throughout, including javadocs since it will -show up in the published javadocs. It should be added to -[AllCookbookExamples.java](https://github.com/marklogic/java-client-api/blob/master/marklogic-client-api/src/main/java/com/marklogic/client/example/cookbook/AllCookbookExamples.java) -in order of recommended examples for developers to review. - -It should have a unit test added to -[this package](https://github.com/marklogic/java-client-api/tree/master/marklogic-client-api/src/test/java/com/marklogic/client/test/example/cookbook). -The unit test can test whatever is needed, however most cookbook unit tests -just run the class and consider it success if no errors are thrown. Some -cookbook examples, such as SSLClientCreator and KerberosClientCreator cannot be -included in unit tests because the unit tests require a server configured with -digest authentication and those tests require a different authentication -scheme. Any cookbook examples not included in unit tests run the risk of -breaking without anyone noticing--hence we have unit tests whenever possible. diff --git a/gradle.properties b/gradle.properties index eddce958e..dc8a5f9c0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,10 +1,11 @@ group=com.marklogic -version=7.2.0 -describedName=MarkLogic Java Client API +version=8.0.0 publishUrl=file:../marklogic-java/releases +okhttpVersion=5.2.0 + # See https://github.com/FasterXML/jackson for more information on the Jackson libraries. -jacksonVersion=2.19.0 +jacksonVersion=2.20.0 # Defined at this level so that they can be set as system properties and used by the marklogic-client-api and test-app # project @@ -18,3 +19,8 @@ testUseReverseProxyServer=false cloudHost= cloudKey= cloudBasePath= + +# See https://docs.gradle.org/current/userguide/toolchains.html#sec:custom_loc for information +# on custom toolchain locations in Gradle. Adding these to try to make Jenkins happy. +org.gradle.java.installations.fromEnv=JAVA_HOME_DIR +org.gradle.java.installations.paths=/home/builder/java/jdk-17.0.2,/home/builder/java/jdk-21.0.1 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1b33c55ba..8bdaf60c7 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d4081da47..2e1113280 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 23d15a936..adff685a0 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -212,7 +210,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" diff --git a/gradlew.bat b/gradlew.bat index 5eed7ee84..e509b2dd8 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,10 @@ goto fail :execute @rem Setup the command line -set CLASSPATH= @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index c12d6085f..d9cebfa63 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -2,75 +2,76 @@ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -test { - useJUnitPlatform() - testLogging{ - events 'started','passed', 'skipped' - } - // For use in testing TestDatabaseClientKerberosFromFile - systemProperty "keytabFile", System.getProperty("keytabFile") - systemProperty "principal", System.getProperty("principal") - - systemProperty "TEST_USE_REVERSE_PROXY_SERVER", testUseReverseProxyServer -} - dependencies { testImplementation project(':marklogic-client-api') + testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation 'org.slf4j:slf4j-api:2.0.17' - testImplementation 'commons-io:commons-io:2.17.0' - testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' + testImplementation 'commons-io:commons-io:2.20.0' + testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" testImplementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" testImplementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" testImplementation "org.jdom:jdom2:2.0.6.1" - testImplementation 'org.apache.commons:commons-lang3:3.18.0' + testImplementation 'org.apache.commons:commons-lang3:3.19.0' + // Allows talking to the Manage API. - testImplementation("com.marklogic:ml-app-deployer:5.0.0") { + testImplementation("com.marklogic:ml-app-deployer:6.0.1") { exclude module: "marklogic-client-api" - // Use the commons-lang3 declared above to keep Black Duck happy. - exclude module: "commons-lang3" } - testImplementation 'ch.qos.logback:logback-classic:1.3.15' + testImplementation 'ch.qos.logback:logback-classic:1.5.18' testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.0' + testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4" } -tasks.register("runFragileTests", Test) { +tasks.withType(Test).configureEach { useJUnitPlatform() - description = "These are called 'fragile' because they'll pass when run by themselves, but when run as part of the " + - "full suite, there seem to be one or more other fast functional tests that run before them and cause some of " + - "their test methods to break. The Jenkinsfile thus calls these first before running the other functional " + - "tests." - include "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class" - include "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class" - include "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class" + + // For use in testing TestDatabaseClientKerberosFromFile + systemProperty "keytabFile", System.getProperty("keytabFile") + systemProperty "principal", System.getProperty("principal") + + systemProperty "TEST_USE_REVERSE_PROXY_SERVER", testUseReverseProxyServer +} + +tasks.register("runFragileTests", Test) { + description = "These are called 'fragile' because they'll pass when run by themselves, but when run as part of the " + + "full suite, there seem to be one or more other fast functional tests that run before them and cause some of " + + "their test methods to break. The Jenkinsfile thus calls these first before running the other functional " + + "tests." + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class" + include "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class" + include "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class" } tasks.register("runFastFunctionalTests", Test) { - useJUnitPlatform() - description = "Run all fast functional tests that don't setup/teardown custom app servers / databases" - include "com/marklogic/client/fastfunctest/**" - // Exclude the "fragile" ones - exclude "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class" - exclude "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class" - exclude "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class" + description = "Run all fast functional tests that don't setup/teardown custom app servers / databases" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include "com/marklogic/client/fastfunctest/**" + // Exclude the "fragile" ones + exclude "com/marklogic/client/fastfunctest/TestQueryOptionBuilder.class" + exclude "com/marklogic/client/fastfunctest/TestRawCombinedQuery.class" + exclude "com/marklogic/client/fastfunctest/TestRawStructuredQuery.class" } tasks.register("runSlowFunctionalTests", Test) { - useJUnitPlatform() - description = "Run slow functional tests; i.e. those that setup/teardown custom app servers / databases" - include "com/marklogic/client/datamovement/functionaltests/**" - include "com/marklogic/client/functionaltest/**" + description = "Run slow functional tests; i.e. those that setup/teardown custom app servers / databases" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include "com/marklogic/client/datamovement/functionaltests/**" + include "com/marklogic/client/functionaltest/**" } tasks.register("runFunctionalTests") { - dependsOn(runFragileTests, runFastFunctionalTests, runSlowFunctionalTests) + dependsOn(runFragileTests, runFastFunctionalTests, runSlowFunctionalTests) } runFastFunctionalTests.mustRunAfter runFragileTests runSlowFunctionalTests.mustRunAfter runFastFunctionalTests diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java index 2f1183a87..22141d798 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java @@ -124,11 +124,10 @@ void invalidPort() { DatabaseClient client = newDatabaseClientBuilder().withPort(assumedInvalidPort).build(); MarkLogicIOException ex = Assertions.assertThrows(MarkLogicIOException.class, () -> client.checkConnection()); - String expected = "Error occurred while calling http://localhost:60123/v1/ping; java.net.ConnectException: " + - "Failed to connect to localhost/127.0.0.1:60123 ; possible reasons for the error include " + + String expected = "Failed to connect to localhost/127.0.0.1:60123 ; possible reasons for the error include " + "that a MarkLogic app server may not be listening on the port, or MarkLogic was stopped " + "or restarted during the request; check the MarkLogic server logs for more information."; - assertEquals(expected, ex.getMessage()); + assertTrue(ex.getMessage().contains(expected), "Unexpected error: " + ex.getMessage()); } @Test diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/BulkIOCallersFnTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/BulkIOCallersFnTest.java index 14bc2d6c2..0c8871c6c 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/BulkIOCallersFnTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/BulkIOCallersFnTest.java @@ -42,7 +42,7 @@ public class BulkIOCallersFnTest extends BasicJavaClientREST { private static String host = null; private static int modulesPort = 8000; - private static int restTestport = 8093; + private static int restTestport = 8015; private static String restServerName = "TestDynamicIngest"; private static SecurityContext secContext = null; diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java index 40c6e17d5..1020c7bd7 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java @@ -15,7 +15,9 @@ import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.FailedRequestException; import com.marklogic.client.admin.ServerConfigurationManager; +import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; import com.marklogic.client.impl.SSLUtil; +import com.marklogic.client.impl.okhttp.RetryIOExceptionInterceptor; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.DocumentMetadataHandle.Capability; import com.marklogic.client.query.QueryManager; @@ -45,6 +47,12 @@ public abstract class ConnectedRESTQA { + static { + DatabaseClientFactory.removeConfigurators(); + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) client -> + client.addInterceptor(new RetryIOExceptionInterceptor(3, 1000, 2, 8000))); + } + private static Properties testProperties = null; private static String authType; diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index b2bfdd46d..a8c48a096 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -3,73 +3,70 @@ */ plugins { - id 'java-library' id 'maven-publish' } -group = 'com.marklogic' - -description = "The official MarkLogic Java client API." - dependencies { - // With 7.0.0, now using the Jakarta JAXB APIs instead of the JAVAX JAXB APIs that were bundled in Java 8. - // To ease support for Java 8, we are depending on version 3.x of the Jakarta JAXB APIs as those only require Java 8, - // whereas the 4.x version requires Java 11 or higher. - api "jakarta.xml.bind:jakarta.xml.bind-api:3.0.1" - implementation "org.glassfish.jaxb:jaxb-runtime:3.0.2" + // Using the latest version now that the 8.0.0 release requires Java 17. + // This is now an implementation dependency as opposed to an api dependency in 7.x and earlier. + // The only time it appears in the public API is when a user uses JAXBHandle. + // But in that scenario, the user would already be using JAXB in their application. + implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + implementation "org.glassfish.jaxb:jaxb-runtime:4.0.6" - implementation 'com.squareup.okhttp3:okhttp:4.12.0' - implementation 'com.squareup.okhttp3:logging-interceptor:4.12.0' - implementation 'io.github.rburgst:okhttp-digest:2.7' + implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" + implementation "com.squareup.okhttp3:logging-interceptor:${okhttpVersion}" + implementation 'io.github.rburgst:okhttp-digest:3.1.1' // We tried upgrading to the org.eclipse.angus:angus-mail dependency, but we ran into significant performance issues // with using the Java Client eval call in our Spark connector. Example - an eval() call for getting 50k URIs would // take 50s instead of 2 to 3s. Haven't dug into the details, but seems like the call isn't lazy and the entire set // of URIs is being retrieved. This implementation - in the old "com.sun.mail" package but still adhering to the new // jakarta.mail API - works fine and performs well for eval calls. - implementation "com.sun.mail:jakarta.mail:2.0.1" + // As of the 8.0.0 release - this still is a good solution, particularly as com.sun.mail:jakarta.mail received a + // recent patch release and is therefore still being maintained. + implementation "com.sun.mail:jakarta.mail:2.0.2" - implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'org.slf4j:slf4j-api:2.0.17' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:${jacksonVersion}" // Only used by extras (which some examples then depend on) compileOnly 'org.jdom:jdom2:2.0.6.1' - compileOnly 'org.dom4j:dom4j:2.1.4' - compileOnly 'com.google.code.gson:gson:2.10.1' + compileOnly 'org.dom4j:dom4j:2.2.0' + compileOnly 'com.google.code.gson:gson:2.13.2' testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' // Forcing junit version to avoid vulnerability with older version in xmlunit testImplementation 'junit:junit:4.13.2' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.0' + testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' testImplementation project(':examples') - testImplementation 'org.apache.commons:commons-lang3:3.18.0' + testImplementation 'org.apache.commons:commons-lang3:3.19.0' + // Allows talking to the Manage API. - testImplementation ("com.marklogic:ml-app-deployer:5.0.0") { + testImplementation("com.marklogic:ml-app-deployer:6.0.1") { exclude module: "marklogic-client-api" - // Use the commons-lang3 declared above to keep Black Duck happy. - exclude module: "commons-lang3" } // Starting with mockito 5.x, Java 11 is required, so sticking with 4.x as we have to support Java 8. testImplementation "org.mockito:mockito-core:4.11.0" testImplementation "org.mockito:mockito-inline:4.11.0" - testImplementation "com.squareup.okhttp3:mockwebserver:4.12.0" + testImplementation "com.squareup.okhttp3:mockwebserver3:5.1.0" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" - testImplementation 'ch.qos.logback:logback-classic:1.3.15' + testImplementation 'ch.qos.logback:logback-classic:1.5.18' // Using this to avoid a schema validation issue with the regular xercesImpl testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' - testImplementation('com.opencsv:opencsv:5.11.2') { + testImplementation('com.opencsv:opencsv:5.12.0') { // Excluding this due to a security vulnerability, and the test for the example that uses this library // passes without this on the classpath. exclude module: "commons-beanutils" } + testImplementation 'org.skyscreamer:jsonassert:1.5.3' // Automatic loading of test framework implementation dependencies is deprecated. @@ -80,8 +77,9 @@ dependencies { } // Ensure that mlHost and mlPassword can override the defaults of localhost/admin if they've been modified -test { +tasks.withType(Test).configureEach { useJUnitPlatform() + systemProperty "TEST_HOST", mlHost systemProperty "TEST_ADMIN_PASSWORD", mlPassword // Needed by the tests for the example programs @@ -90,143 +88,104 @@ test { systemProperty "TEST_USE_REVERSE_PROXY_SERVER", testUseReverseProxyServer } -task sourcesJar(type: Jar) { - archiveClassifier = 'sources' - exclude ('property', '*.xsd', '*.xjb') - from sourceSets.main.allSource -} - -javadoc { - maxMemory="6000m" - options.overview = "src/main/javadoc/overview.html" - options.windowTitle = "$rootProject.describedName $rootProject.version" - options.docTitle = "$rootProject.describedName $rootProject.version" - options.bottom = "Copyright © 2024 MarkLogic Corporation. All Rights Reserved." - options.links = [ 'http://docs.oracle.com/javase/8/docs/api/' ] - options.use = true - if (JavaVersion.current().isJava9Compatible()) { - options.addBooleanOption('html4', true) - } - exclude([ - '**/impl/**', '**/jaxb/**', '**/test/**' - ]) -// workaround for bug in options.docFilesSubDirs = true - doLast{ - copy{ - from "${projectDir}/src/main/javadoc/doc-files" - into "${buildDir}/docs/javadoc/doc-files" - } - } -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - archiveClassifier = 'javadoc' - from javadoc.destinationDir -} - -Node pomCustomizations = new NodeBuilder(). project { - name "$rootProject.describedName" - packaging 'jar' - textdescription "$project.description" - url 'https://github.com/marklogic/java-client-api' - - scm { - url 'git@github.com:marklogic/java-client-api.git' - connection 'scm:git:git@github.com:marklogic/java-client-api.git' - developerConnection 'scm:git:git@github.com:marklogic/java-client-api.git' - } - - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - - developers { - developer { - name 'MarkLogic' - email 'java-sig@marklogic.com' - organization 'MarkLogic' - organizationUrl 'https://www.marklogic.com' - } - developer { - name 'MarkLogic Github Contributors' - email 'general@developer.marklogic.com' - organization 'Github Contributors' - organizationUrl 'https://github.com/marklogic/java-client-api/graphs/contributors' - } - } -} - -publishing { - publications { - mainJava(MavenPublication) { - from components.java - - pom.withXml { - asNode().append(pomCustomizations.packaging) - asNode().append(pomCustomizations.name) - asNode().appendNode("description", pomCustomizations.textdescription.text()) - asNode().append(pomCustomizations.url) - asNode().append(pomCustomizations.licenses) - asNode().append(pomCustomizations.developers) - asNode().append(pomCustomizations.scm) - } - artifact sourcesJar - artifact javadocJar - } - } - repositories { - maven { - if(project.hasProperty("mavenUser")) { - credentials { - username mavenUser - password mavenPassword - } - } - url = publishUrl - } - } -} - -task printClassPath() { - doLast { - println sourceSets.main.runtimeClasspath.asPath+':'+sourceSets.test.runtimeClasspath.asPath - } -} - -task generatePomForDependencyGraph(dependsOn: "generatePomFileForMainJavaPublication") { - description = "Prepare for a release by making a copy of the generated pom file in the root directory so that it " + - "can enable Github's Dependency Graph feature, which does not yet support Gradle" - doLast { - def preamble = '' - def comment = "" - def fileText = file("build/publications/mainJava/pom-default.xml").getText() - file("../pom.xml").setText(fileText.replace(preamble, preamble + comment)) - } -} - -task testRows(type: Test) { - useJUnitPlatform() +tasks.register("testRows", Test) { description = "Run all 'rows' tests; i.e. those exercising Optic and Optic Update functionality" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath include "com/marklogic/client/test/rows/**" } -task debugCloudAuth(type: JavaExec) { +tasks.register("debugCloudAuth", JavaExec) { description = "Test program for manual testing of cloud-based authentication against a Progress Data Cloud instance" - mainClass = 'com.marklogic.client.test.MarkLogicCloudAuthenticationDebugger' + mainClass = 'com.marklogic.client.test.ProgressDataCloudAuthenticationDebugger' classpath = sourceSets.test.runtimeClasspath args = [cloudHost, cloudKey, cloudBasePath] } -task runXmlSmokeTests(type: Test) { +tasks.register("runXmlSmokeTests", Test) { description = "Run a bunch of XML-related tests for smoke-testing on a particular JVM" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath include "com/marklogic/client/test/BufferableHandleTest.class" include "com/marklogic/client/test/EvalTest.class" include "com/marklogic/client/test/HandleAsTest.class" include "com/marklogic/client/test/JAXBHandleTest.class" } + +// Publishing setup - see https://docs.gradle.org/current/userguide/publishing_setup.html . +java { + withJavadocJar() + withSourcesJar() +} + +javadoc { + maxMemory = "6000m" + options.overview = "src/main/javadoc/overview.html" + options.windowTitle = "MarkLogic Java Client API $rootProject.version" + options.docTitle = "MarkLogic Java Client API $rootProject.version" + options.bottom = "Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved." + options.use = true + exclude([ + '**/impl/**', '**/jaxb/**', '**/test/**' + ]) +// workaround for bug in options.docFilesSubDirs = true + doLast { + copy { + from "${projectDir}/src/main/javadoc/doc-files" + into "${layout.buildDirectory.get()}/docs/javadoc/doc-files" + } + } +} + +publishing { + publications { + mainJava(MavenPublication) { + from components.java + pom { + name = "${project.group}:${project.name}" + description = "The MarkLogic Java Client API" + packaging = "jar" + url = "https://github.com/marklogic/java-client-api" + licenses { + license { + name = "The Apache License, Version 2.0" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + id = "marklogic" + name = "MarkLogic Github Contributors" + email = "general@developer.marklogic.com" + organization = "MarkLogic" + organizationUrl = "https://www.marklogic.com" + } + } + scm { + url = "https://github.com/marklogic/java-client-api" + connection = "https://github.com/marklogic/java-client-api" + developerConnection = "https://github.com/marklogic/java-client-api" + } + } + } + } + repositories { + maven { + if (project.hasProperty("mavenUser")) { + credentials { + username = mavenUser + password = mavenPassword + } + url = publishUrl + allowInsecureProtocol = true + } else { + name = "central" + url = mavenCentralUrl + credentials { + username = mavenCentralUsername + password = mavenCentralPassword + } + } + } + } +} diff --git a/marklogic-client-api/gradle.properties b/marklogic-client-api/gradle.properties new file mode 100644 index 000000000..4bee29e3b --- /dev/null +++ b/marklogic-client-api/gradle.properties @@ -0,0 +1,8 @@ +# Define these on the command line to publish to OSSRH +# See https://central.sonatype.org/publish/publish-gradle/#credentials for more information +mavenCentralUsername= +mavenCentralPassword= +mavenCentralUrl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ +#signing.keyId=YourKeyId +#signing.password=YourPublicKeyPassword +#signing.secretKeyRingFile=PathToYourKeyRingFile diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/ClientCookie.java b/marklogic-client-api/src/main/java/com/marklogic/client/ClientCookie.java new file mode 100644 index 000000000..810b4bbb4 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/ClientCookie.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client; + +import java.util.concurrent.TimeUnit; + +/** + * ClientCookie is a wrapper around the Cookie implementation so that the + * underlying implementation can be changed. + * + */ +public class ClientCookie { + + private final String name; + private final String value; + private final long expiresAt; + private final String domain; + private final String path; + private final boolean secure; + + public ClientCookie(String name, String value, long expiresAt, String domain, String path, boolean secure) { + this.name = name; + this.value = value; + this.expiresAt = expiresAt; + this.domain = domain; + this.path = path; + this.secure = secure; + } + + public boolean isSecure() { + return secure; + } + + public String getPath() { + return path; + } + + public String getDomain() { + return domain; + } + + public long expiresAt() { + return expiresAt; + } + + public String getName() { + return name; + } + + public int getMaxAge() { + return (int) TimeUnit.MILLISECONDS.toSeconds(expiresAt - System.currentTimeMillis()); + } + + public String getValue() { + return value; + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java index 8406d7213..63e4cd212 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java @@ -7,7 +7,6 @@ import com.marklogic.client.impl.*; import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.io.marker.ContentHandleFactory; -import okhttp3.OkHttpClient; import javax.naming.InvalidNameException; import javax.naming.ldap.LdapName; @@ -31,7 +30,7 @@ */ public class DatabaseClientFactory { - static private List> clientConfigurators = Collections.synchronizedList(new ArrayList<>()); + static private List clientConfigurators = Collections.synchronizedList(new ArrayList<>()); static private HandleFactoryRegistry handleRegistry = HandleFactoryRegistryImpl.newDefault(); @@ -349,70 +348,7 @@ public SecurityContext withSSLContext(SSLContext context, X509TrustManager trust } /** - * @since 6.1.0 - * @deprecated as of 7.2.0; use {@code ProgressDataCloudAuthContext} instead. Will be removed in 8.0.0. - */ - @Deprecated - public static class MarkLogicCloudAuthContext extends ProgressDataCloudAuthContext { - - /** - * @param apiKey user's API key for accessing Progress Data Cloud - */ - public MarkLogicCloudAuthContext(String apiKey) { - super(apiKey); - } - - /** - * @param apiKey user's API key for accessing Progress Data Cloud - * @param tokenDuration length in minutes until the generated access token expires - * @since 6.3.0 - */ - public MarkLogicCloudAuthContext(String apiKey, Integer tokenDuration) { - super(apiKey, tokenDuration); - } - - /** - * Only intended to be used in the scenario that the token endpoint of "/token" and the grant type of "apikey" - * are not the intended values. - * - * @param apiKey user's API key for accessing Progress Data Cloud - * @param tokenEndpoint for overriding the default token endpoint if necessary - * @param grantType for overriding the default grant type if necessary - */ - public MarkLogicCloudAuthContext(String apiKey, String tokenEndpoint, String grantType) { - super(apiKey, tokenEndpoint, grantType); - } - - /** - * Only intended to be used in the scenario that the token endpoint of "/token" and the grant type of "apikey" - * are not the intended values. - * - * @param apiKey user's API key for accessing Progress Data Cloud - * @param tokenEndpoint for overriding the default token endpoint if necessary - * @param grantType for overriding the default grant type if necessary - * @param tokenDuration length in minutes until the generated access token expires - * @since 6.3.0 - */ - public MarkLogicCloudAuthContext(String apiKey, String tokenEndpoint, String grantType, Integer tokenDuration) { - super(apiKey, tokenEndpoint, grantType, tokenDuration); - } - - @Override - public MarkLogicCloudAuthContext withSSLContext(SSLContext context, X509TrustManager trustManager) { - this.sslContext = context; - this.trustManager = trustManager; - return this; - } - - @Override - public MarkLogicCloudAuthContext withSSLHostnameVerifier(SSLHostnameVerifier verifier) { - this.sslVerifier = verifier; - return this; - } - } - - /** - * @since 7.2.0 Use this instead of the now-deprecated {@code MarkLogicCloudAuthContext} + * @since 7.2.0 Replaced {@code MarkLogicCloudAuthContext} which was removed in 8.0.0 */ public static class ProgressDataCloudAuthContext extends AuthContext { private String tokenEndpoint; @@ -1329,33 +1265,17 @@ static public DatabaseClient newClient(String host, int port, String database, static public DatabaseClient newClient(String host, int port, String basePath, String database, SecurityContext securityContext, DatabaseClient.ConnectionType connectionType) { - RESTServices services = new OkHttpServices(); // As of 6.1.0, the following optimization is made as it's guaranteed that if the user is connecting to a // Progress Data Cloud instance, then port 443 will be used. Every path for constructing a DatabaseClient goes through // this method, ensuring that this optimization will always be applied, and thus freeing the user from having to // worry about what port to configure when using Progress Data Cloud. - if (securityContext instanceof MarkLogicCloudAuthContext || securityContext instanceof ProgressDataCloudAuthContext) { + if (securityContext instanceof ProgressDataCloudAuthContext) { port = 443; } - services.connect(host, port, basePath, database, securityContext); - - if (clientConfigurators != null) { - clientConfigurators.forEach(configurator -> { - if (configurator instanceof OkHttpClientConfigurator) { - OkHttpClient okHttpClient = (OkHttpClient) services.getClientImplementation(); - Objects.requireNonNull(okHttpClient); - OkHttpClient.Builder clientBuilder = okHttpClient.newBuilder(); - ((OkHttpClientConfigurator) configurator).configure(clientBuilder); - ((OkHttpServices) services).setClientImplementation(clientBuilder.build()); - } else { - throw new IllegalArgumentException("A ClientConfigurator must implement OkHttpClientConfigurator"); - } - }); - } - DatabaseClientImpl client = new DatabaseClientImpl( - services, host, port, basePath, database, securityContext, connectionType - ); + OkHttpServices.ConnectionConfig config = new OkHttpServices.ConnectionConfig(host, port, basePath, database, securityContext, clientConfigurators); + RESTServices services = new OkHttpServices(config); + DatabaseClientImpl client = new DatabaseClientImpl(services, host, port, basePath, database, securityContext, connectionType); client.setHandleRegistry(getHandleRegistry().copy()); return client; } @@ -1397,13 +1317,13 @@ static public void registerDefaultHandles() { * @param configurator the listener for configuring the communication library */ static public void addConfigurator(ClientConfigurator configurator) { - if (!OkHttpClientConfigurator.class.isInstance(configurator)) { - throw new IllegalArgumentException( - "Configurator must implement OkHttpClientConfigurator" - ); - } + if (!OkHttpClientConfigurator.class.isInstance(configurator)) { + throw new IllegalArgumentException( + "Configurator must implement OkHttpClientConfigurator" + ); + } - clientConfigurators.add(configurator); + clientConfigurators.add((OkHttpClientConfigurator) configurator); } /** diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/Transaction.java b/marklogic-client-api/src/main/java/com/marklogic/client/Transaction.java index 578743845..d98071bef 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/Transaction.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/Transaction.java @@ -3,7 +3,6 @@ */ package com.marklogic.client; -import com.marklogic.client.impl.ClientCookie; import com.marklogic.client.io.marker.StructureReadHandle; import java.util.List; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/extra/gson/GSONHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/extra/gson/GSONHandle.java index 511bf4163..dfdbbbb90 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/extra/gson/GSONHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/extra/gson/GSONHandle.java @@ -3,12 +3,6 @@ */ package com.marklogic.client.extra.gson; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; - import com.google.gson.JsonElement; import com.google.gson.JsonIOException; import com.google.gson.JsonParser; @@ -16,9 +10,14 @@ import com.marklogic.client.MarkLogicIOException; import com.marklogic.client.io.BaseHandle; import com.marklogic.client.io.Format; -import com.marklogic.client.io.marker.ResendableContentHandle; import com.marklogic.client.io.marker.*; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + /** * A GSONHandle represents JSON content as a GSON JsonElement for reading or * writing. You must install the GSON library to use this class. @@ -83,18 +82,6 @@ public GSONHandle[] newHandleArray(int length) { return new GSONHandle[length]; } - /** - * Returns the parser used to construct element objects from JSON. - * @return the JSON parser. - * @deprecated Use static methods like JsonParser.parseString() or JsonParser.parseReader() directly instead - */ - @Deprecated - public JsonParser getParser() { - if (parser == null) - parser = new JsonParser(); - return parser; - } - /** * Returns the root node of the JSON tree. * @return the JSON root element. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/extra/okhttpclient/OkHttpClientBuilderFactory.java b/marklogic-client-api/src/main/java/com/marklogic/client/extra/okhttpclient/OkHttpClientBuilderFactory.java index fcd0c022f..995a5392b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/extra/okhttpclient/OkHttpClientBuilderFactory.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/extra/okhttpclient/OkHttpClientBuilderFactory.java @@ -7,6 +7,8 @@ import com.marklogic.client.impl.okhttp.OkHttpUtil; import okhttp3.OkHttpClient; +import java.util.ArrayList; + /** * Exposes the mechanism for constructing an {@code OkHttpClient.Builder} in the same fashion as when a * {@code DatabaseClient} is constructed. Primarily intended for reuse in the ml-app-deployer library. If the @@ -17,6 +19,6 @@ public interface OkHttpClientBuilderFactory { static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseClientFactory.SecurityContext securityContext) { - return OkHttpUtil.newOkHttpClientBuilder(host, securityContext); + return OkHttpUtil.newOkHttpClientBuilder(host, securityContext, new ArrayList<>()); } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/ClientCookie.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/ClientCookie.java deleted file mode 100644 index 48735bd98..000000000 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/ClientCookie.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. - */ -package com.marklogic.client.impl; - -import java.util.concurrent.TimeUnit; - -import okhttp3.Cookie; -import okhttp3.HttpUrl; - -/** - * ClientCookie is a wrapper around the Cookie implementation so that the - * underlying implementation can be changed. - * - */ -public class ClientCookie { - Cookie cookie; - - ClientCookie(String name, String value, long expiresAt, String domain, String path, - boolean secure) { - Cookie.Builder cookieBldr = new Cookie.Builder() - .domain(domain) - .path(path) - .name(name) - .value(value) - .expiresAt(expiresAt); - if ( secure == true ) cookieBldr = cookieBldr.secure(); - this.cookie = cookieBldr.build(); - } - - public ClientCookie(ClientCookie cookie) { - this(cookie.getName(), cookie.getValue(), cookie.expiresAt(), cookie.getDomain(), cookie.getPath(), - cookie.isSecure()); - } - - public boolean isSecure() { - return cookie.secure(); - } - - public String getPath() { - return cookie.path(); - } - - public String getDomain() { - return cookie.domain(); - } - - public long expiresAt() { - return cookie.expiresAt(); - } - - public String getName() { - return cookie.name(); - } - - public int getMaxAge() { - return (int) TimeUnit.MILLISECONDS.toSeconds(cookie.expiresAt() - System.currentTimeMillis()); - } - public String getValue() { - return cookie.value(); - } - - public static ClientCookie parse(HttpUrl url, String setCookie) { - Cookie cookie = Cookie.parse(url, setCookie); - if(cookie == null) throw new IllegalStateException(setCookie + "is not a well-formed cookie"); - return new ClientCookie(cookie.name(), cookie.value(), cookie.expiresAt(), cookie.domain(), cookie.path(), - cookie.secure()); - } - - public String toString() { - return cookie.toString(); - } -} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientImpl.java index a61b7df47..7a144b2dd 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientImpl.java @@ -60,8 +60,6 @@ public DatabaseClientImpl(RESTServices services, String host, int port, String b this.database = database; this.securityContext = securityContext; this.connectionType = connectionType; - - services.setDatabaseClient(this); } public long getServerVersion() { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java index 5b0d2cb5a..00ca7c3be 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java @@ -11,7 +11,6 @@ import javax.xml.parsers.ParserConfigurationException; import com.marklogic.client.io.Format; -import okhttp3.MediaType; import org.w3c.dom.Document; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index dd8a81bc0..cd7bf9e6f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -18,9 +18,11 @@ import com.marklogic.client.document.DocumentManager.Metadata; import com.marklogic.client.eval.EvalResult; import com.marklogic.client.eval.EvalResultIterator; +import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; import com.marklogic.client.impl.okhttp.HttpUrlBuilder; import com.marklogic.client.impl.okhttp.OkHttpUtil; import com.marklogic.client.impl.okhttp.PartIterator; +import com.marklogic.client.impl.okhttp.RetryableRequestBody; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.*; import com.marklogic.client.query.*; @@ -74,10 +76,10 @@ public class OkHttpServices implements RESTServices { static final private Logger logger = LoggerFactory.getLogger(OkHttpServices.class); - static final public String OKHTTP_LOGGINGINTERCEPTOR_LEVEL = "com.marklogic.client.okhttp.httplogginginterceptor.level"; - static final public String OKHTTP_LOGGINGINTERCEPTOR_OUTPUT = "com.marklogic.client.okhttp.httplogginginterceptor.output"; + private static final String OKHTTP_LOGGINGINTERCEPTOR_LEVEL = "com.marklogic.client.okhttp.httplogginginterceptor.level"; + private static final String OKHTTP_LOGGINGINTERCEPTOR_OUTPUT = "com.marklogic.client.okhttp.httplogginginterceptor.output"; - static final private String DOCUMENT_URI_PREFIX = "/documents?uri="; + private static final String DOCUMENT_URI_PREFIX = "/documents?uri="; static final private int DELAY_FLOOR = 125; static final private int DELAY_CEILING = 2000; @@ -88,21 +90,29 @@ public class OkHttpServices implements RESTServices { private final static MediaType URLENCODED_MIME_TYPE = MediaType.parse("application/x-www-form-urlencoded; charset=UTF-8"); private final static String UTF8_ID = StandardCharsets.UTF_8.toString(); - private DatabaseClient databaseClient; private String database = null; private HttpUrl baseUri; - private OkHttpClient client; + + // This should really be final, but given the history of this class and the former "connect()" method that meant + // the client was created in the constructor, this is being kept as non-final so it can be assigned a value of null + // on release. + private OkHttpClient okHttpClient; + private boolean released = false; + /** + * The next 4 fields implement an application-level retry that only works for certain HTTP status codes. It will not + * attempt a retry on any IOException or any type of connection failure. Sadly, the logic that uses these fields is + * in several places and is slightly different in each place. It's also not possible to implement this logic in an + * OkHttp interceptor as the logic needs access to details that are not available to an interceptor. + */ private final Random randRetry = new Random(); - private int maxDelay = DEFAULT_MAX_DELAY; private int minRetry = DEFAULT_MIN_RETRY; + private final Set retryStatus = new HashSet<>(); private boolean checkFirstRequest = true; - private final Set retryStatus = new HashSet<>(); - static protected class ThreadState { boolean isFirstRequest; @@ -114,25 +124,23 @@ static protected class ThreadState { private final ThreadLocal threadState = ThreadLocal.withInitial(() -> new ThreadState(checkFirstRequest)); - public OkHttpServices() { + public record ConnectionConfig(String host, int port, String basePath, String database, + SecurityContext securityContext, List clientConfigurators) { + } + + public OkHttpServices(ConnectionConfig connectionConfig) { retryStatus.add(STATUS_BAD_GATEWAY); retryStatus.add(STATUS_SERVICE_UNAVAILABLE); retryStatus.add(STATUS_GATEWAY_TIMEOUT); - } - @Override - public Set getRetryStatus() { - return retryStatus; + this.okHttpClient = connect(connectionConfig); } - @Override - public int getMaxDelay() { - return maxDelay; - } - - @Override - public void setMaxDelay(int maxDelay) { - this.maxDelay = maxDelay; + private static ClientCookie parseClientCookie(HttpUrl url, String setCookieHeaderValue) { + Cookie cookie = Cookie.parse(url, setCookieHeaderValue); + if(cookie == null) throw new IllegalStateException(setCookieHeaderValue + " is not a well-formed cookie"); + return new ClientCookie(cookie.name(), cookie.value(), cookie.expiresAt(), cookie.domain(), cookie.path(), + cookie.secure()); } private FailedRequest extractErrorFields(Response response) { @@ -176,18 +184,19 @@ private FailedRequest extractErrorFields(Response response) { } } - @Override - public void connect(String host, int port, String basePath, String database, SecurityContext securityContext) { - if (host == null) + private OkHttpClient connect(ConnectionConfig config) { + if (config.host == null) { throw new IllegalArgumentException("No host provided"); - if (securityContext == null) + } + if (config.securityContext == null) { throw new IllegalArgumentException("No security context provided"); + } - this.checkFirstRequest = securityContext instanceof DigestAuthContext; - this.database = database; - this.baseUri = HttpUrlBuilder.newBaseUrl(host, port, basePath, securityContext.getSSLContext()); + this.checkFirstRequest = config.securityContext instanceof DigestAuthContext; + this.database = config.database; + this.baseUri = HttpUrlBuilder.newBaseUrl(config.host, config.port, config.basePath, config.securityContext.getSSLContext()); - OkHttpClient.Builder clientBuilder = OkHttpUtil.newOkHttpClientBuilder(host, securityContext); + OkHttpClient.Builder clientBuilder = OkHttpUtil.newOkHttpClientBuilder(config.host, config.securityContext, config.clientConfigurators); Properties props = System.getProperties(); if (props.containsKey(OKHTTP_LOGGINGINTERCEPTOR_LEVEL)) { @@ -195,15 +204,12 @@ public void connect(String host, int port, String basePath, String database, Sec } this.configureDelayAndRetry(props); - this.client = clientBuilder.build(); + return clientBuilder.build(); } /** * Based on the given properties, add a network interceptor to the given OkHttpClient.Builder to log HTTP * traffic. - * - * @param clientBuilder - * @param props */ private void configureOkHttpLogging(OkHttpClient.Builder clientBuilder, Properties props) { final boolean useLogger = "LOGGER".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_OUTPUT)); @@ -244,40 +250,21 @@ private void configureDelayAndRetry(Properties props) { } } - @Override - public DatabaseClient getDatabaseClient() { - return databaseClient; - } - - @Override - public void setDatabaseClient(DatabaseClient client) { - this.databaseClient = client; - } - - private OkHttpClient getConnection() { - if (client != null) { - return client; - } else if (released) { - throw new IllegalStateException( - "You cannot use this connected object anymore--connection has already been released"); - } else { - throw new MarkLogicInternalException("Cannot proceed--connection is null for unknown reason"); - } - } - @Override public void release() { - if (client == null) return; + if (released || okHttpClient == null) { + return; + } try { released = true; - client.dispatcher().executorService().shutdownNow(); + okHttpClient.dispatcher().executorService().shutdownNow(); } finally { try { - if (client.cache() != null) client.cache().close(); + if (okHttpClient.cache() != null) okHttpClient.cache().close(); } catch (IOException e) { throw new MarkLogicIOException(e); } finally { - client = null; + okHttpClient = null; logger.debug("Releasing connection"); } } @@ -491,8 +478,13 @@ private Response sendRequestOnce(Request.Builder requestBldr) { } private Response sendRequestOnce(Request request) { + if (released) { + throw new IllegalStateException( + "You cannot use this connected object anymore--connection has already been released"); + } + try { - return getConnection().newCall(request).execute(); + return okHttpClient.newCall(request).execute(); } catch (IOException e) { if (e instanceof SSLException) { String message = e.getMessage(); @@ -1521,7 +1513,7 @@ public Response apply(Request.Builder funcBuilder) { String location = response.headers().get("Location"); List cookies = new ArrayList<>(); for (String setCookie : response.headers(HEADER_SET_COOKIE)) { - ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); + ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); cookies.add(cookie); } closeResponse(response); @@ -2591,25 +2583,6 @@ public Response apply(Request.Builder funcBuilder) { return (reqlog != null) ? reqlog.copyContent(entity) : entity; } - @Override - public void postValue(RequestLogger reqlog, String type, String key, - String mimetype, Object value) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException { - logger.debug("Posting {}/{}", type, key); - - putPostValueImpl(reqlog, "post", type, key, null, mimetype, value, STATUS_CREATED); - } - - @Override - public void postValue(RequestLogger reqlog, String type, String key, - RequestParameters extraParams) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException { - logger.debug("Posting {}/{}", type, key); - - putPostValueImpl(reqlog, "post", type, key, extraParams, null, null, STATUS_NO_CONTENT); - } - - @Override public void putValue(RequestLogger reqlog, String type, String key, String mimetype, Object value) @@ -2795,42 +2768,6 @@ public Response apply(Request.Builder funcBuilder) { logRequest(reqlog, "deleted %s value with %s key", type, key); } - @Override - public void deleteValues(RequestLogger reqlog, String type) - throws ForbiddenUserException, FailedRequestException { - logger.debug("Deleting {}", type); - - Request.Builder requestBldr = setupRequest(type, null); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.delete().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to delete " - + type, extractErrorFields(response)); - } - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("delete failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - closeResponse(response); - - logRequest(reqlog, "deleted %s values", type); - } - - @Override - public R getSystemSchema(RequestLogger reqlog, String schemaName, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { - RequestParameters params = new RequestParameters(); - params.add("system", schemaName); - return getResource(reqlog, "internal/schemas", null, params, output); - } - @Override public R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output @@ -3352,7 +3289,7 @@ public R postResou } @Override - public R postBulkDocuments( + public void postBulkDocuments( RequestLogger reqlog, DocumentWriteSet writeSet, ServerTransform transform, Transaction transaction, Format defaultFormat, R output, String temporalCollection, String extraContentDispositionParams) @@ -3411,7 +3348,7 @@ public R postBulkDocuments( transform.merge(params); } if (temporalCollection != null) params.add("temporal-collection", temporalCollection); - return postResource(reqlog, "documents", transaction, params, + postResource(reqlog, "documents", transaction, params, (AbstractWriteHandle[]) writeHandles.toArray(new AbstractWriteHandle[0]), (RequestParameters[]) headerList.toArray(new RequestParameters[0]), output); @@ -4843,12 +4780,7 @@ public T getContentAs(Class as) { @Override public OkHttpClient getClientImplementation() { - if (client == null) return null; - return client; - } - - public void setClientImplementation(OkHttpClient client) { - this.client = client; + return okHttpClient; } @Override @@ -5153,12 +5085,12 @@ public R getGraphUris(RequestLogger reqlog, R out } @Override - public R readGraph(RequestLogger reqlog, String uri, R output, + public void readGraph(RequestLogger reqlog, String uri, R output, Transaction transaction) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { RequestParameters params = new RequestParameters(); addGraphUriParam(params, uri); - return getResource(reqlog, "graphs", transaction, params, output); + getResource(reqlog, "graphs", transaction, params, output); } @Override @@ -5235,12 +5167,11 @@ public void mergePermissions(RequestLogger reqlog, String uri, } @Override - public Object deleteGraph(RequestLogger reqlog, String uri, Transaction transaction) + public void deleteGraph(RequestLogger reqlog, String uri, Transaction transaction) throws ForbiddenUserException, FailedRequestException { RequestParameters params = new RequestParameters(); addGraphUriParam(params, uri); - return deleteResource(reqlog, "graphs", transaction, params, null); - + deleteResource(reqlog, "graphs", transaction, params, null); } @Override @@ -5482,7 +5413,8 @@ static private List getPartList(MimeMultipart multipart) { } } - static private class ObjectRequestBody extends RequestBody { + static private class ObjectRequestBody extends RequestBody implements RetryableRequestBody { + private Object obj; private MediaType contentType; @@ -5516,6 +5448,13 @@ public void writeTo(BufferedSink sink) throws IOException { throw new IllegalStateException("Cannot write object of type: " + obj.getClass()); } } + + @Override + public boolean isRetryable() { + // Added in 8.0.0 to work with the retry interceptor so it knows whether the body can be retried or not. + // InputStreams cannot be retried as they are consumed on first read. + return !(obj instanceof InputStream); + } } // API First Changes @@ -5680,7 +5619,7 @@ private void executeRequest(CallResponseImpl responseImpl) { if (session != null) { List cookies = new ArrayList<>(); for (String setCookie : response.headers(HEADER_SET_COOKIE)) { - ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); + ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); cookies.add(cookie); } ((SessionStateImpl) session).setCookies(cookies); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java index 7750361c6..0f643daa7 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java @@ -3,34 +3,15 @@ */ package com.marklogic.client.impl; -import java.io.InputStream; -import java.io.Reader; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; - -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.DatabaseClientFactory.SecurityContext; -import com.marklogic.client.FailedRequestException; -import com.marklogic.client.ForbiddenUserException; -import com.marklogic.client.ResourceNotFoundException; -import com.marklogic.client.ResourceNotResendableException; -import com.marklogic.client.SessionState; -import com.marklogic.client.Transaction; +import com.marklogic.client.DatabaseClient.ConnectionResult; +import com.marklogic.client.*; import com.marklogic.client.bitemporal.TemporalDescriptor; import com.marklogic.client.bitemporal.TemporalDocumentManager.ProtectionLevel; -import com.marklogic.client.document.DocumentDescriptor; +import com.marklogic.client.document.*; import com.marklogic.client.document.DocumentManager.Metadata; -import com.marklogic.client.document.DocumentPage; -import com.marklogic.client.document.DocumentUriTemplate; -import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.document.ServerTransform; import com.marklogic.client.eval.EvalResultIterator; import com.marklogic.client.extensions.ResourceServices.ServiceResult; import com.marklogic.client.extensions.ResourceServices.ServiceResultIterator; -import com.marklogic.client.DatabaseClient.ConnectionResult; import com.marklogic.client.io.BytesHandle; import com.marklogic.client.io.Format; import com.marklogic.client.io.InputStreamHandle; @@ -44,6 +25,14 @@ import com.marklogic.client.util.RequestLogger; import com.marklogic.client.util.RequestParameters; +import java.io.InputStream; +import java.io.Reader; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + public interface RESTServices { String AUTHORIZATION_TYPE_SAML = "SAML"; @@ -78,7 +67,6 @@ public interface RESTServices { String MIMETYPE_APPLICATION_JSON = "application/json"; String MIMETYPE_APPLICATION_XML = "application/xml"; String MIMETYPE_MULTIPART_MIXED = "multipart/mixed"; - String MIMETYPE_MULTIPART_FORM = "multipart/form-data"; int STATUS_OK = 200; int STATUS_CREATED = 201; @@ -98,13 +86,6 @@ public interface RESTServices { String MAX_DELAY_PROP = "com.marklogic.client.maximumRetrySeconds"; String MIN_RETRY_PROP = "com.marklogic.client.minimumRetries"; - Set getRetryStatus(); - int getMaxDelay(); - void setMaxDelay(int maxDelay); - - void connect(String host, int port, String basePath, String database, SecurityContext securityContext); - DatabaseClient getDatabaseClient(); - void setDatabaseClient(DatabaseClient client); void release(); TemporalDescriptor deleteDocument(RequestLogger logger, DocumentDescriptor desc, Transaction transaction, @@ -129,7 +110,7 @@ DocumentPage getBulkDocuments(RequestLogger logger, long serverTimestamp, Search RequestParameters extraParams, String forestName) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; - T postBulkDocuments(RequestLogger logger, DocumentWriteSet writeSet, + void postBulkDocuments(RequestLogger logger, DocumentWriteSet writeSet, ServerTransform transform, Transaction transaction, Format defaultFormat, T output, String temporalCollection, String extraContentDispositionParams) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; @@ -188,10 +169,6 @@ T getValues(RequestLogger logger, String type, String mimetype, Class as) T getValues(RequestLogger reqlog, String type, RequestParameters extraParams, String mimetype, Class as) throws ForbiddenUserException, FailedRequestException; - void postValue(RequestLogger logger, String type, String key, String mimetype, Object value) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException; - void postValue(RequestLogger reqlog, String type, String key, RequestParameters extraParams) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException; void putValue(RequestLogger logger, String type, String key, String mimetype, Object value) throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, @@ -202,11 +179,6 @@ void putValue(RequestLogger logger, String type, String key, RequestParameters e FailedRequestException; void deleteValue(RequestLogger logger, String type, String key) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; - void deleteValues(RequestLogger logger, String type) - throws ForbiddenUserException, FailedRequestException; - - R getSystemSchema(RequestLogger reqlog, String schemaName, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output) @@ -335,7 +307,7 @@ public boolean isExpected(int status) { R getGraphUris(RequestLogger reqlog, R output) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; - R readGraph(RequestLogger reqlog, String uri, R output, + void readGraph(RequestLogger reqlog, String uri, R output, Transaction transaction) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; void writeGraph(RequestLogger reqlog, String uri, @@ -343,7 +315,7 @@ void writeGraph(RequestLogger reqlog, String uri, throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; void writeGraphs(RequestLogger reqlog, AbstractWriteHandle input, Transaction transaction) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; - Object deleteGraph(RequestLogger requestLogger, String uri, + void deleteGraph(RequestLogger requestLogger, String uri, Transaction transaction) throws ForbiddenUserException, FailedRequestException; void deleteGraphs(RequestLogger requestLogger, Transaction transaction) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestParametersImplementation.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestParametersImplementation.java index 7cdabe9e7..5dadfdd72 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestParametersImplementation.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestParametersImplementation.java @@ -3,22 +3,24 @@ */ package com.marklogic.client.impl; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import javax.ws.rs.core.AbstractMultivaluedMap; -import javax.ws.rs.core.MultivaluedMap; public abstract class RequestParametersImplementation { - private MultivaluedMap map = - new AbstractMultivaluedMap(new ConcurrentHashMap<>()) {}; - protected RequestParametersImplementation() { - super(); - } + // Prior to 8.0.0, this was a threadsafe map. However, that fact was not documented for a user. And in practice, + // it would not make sense for multiple threads to share a mutable instance of this, or of one of its subclasses. + // Additionally, the impl was from the 'javax.ws.rs:javax.ws.rs-api:2.1.1' dependency which wasn't used for + // anything else. So for 8.0.0, this is now simply a map that matches the intended usage of this class and its + // subclasses, which is to be used by a single thread. + private final Map> map = new HashMap<>(); + + protected RequestParametersImplementation() { + super(); + } - protected Map> getMap() { - return map; - } + protected Map> getMap() { + return map; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/SessionStateImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/SessionStateImpl.java index 53d5defe9..1c27cccfb 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/SessionStateImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/SessionStateImpl.java @@ -3,6 +3,7 @@ */ package com.marklogic.client.impl; +import com.marklogic.client.ClientCookie; import com.marklogic.client.SessionState; import java.util.ArrayList; @@ -12,43 +13,42 @@ import java.util.concurrent.atomic.AtomicBoolean; public class SessionStateImpl implements SessionState { - private List cookies; - private String sessionId; - private AtomicBoolean setCreatedTimestamp; - private Calendar created; - - public SessionStateImpl() { - sessionId = Long.toUnsignedString(ThreadLocalRandom.current().nextLong(), 16); - cookies = new ArrayList<>(); - setCreatedTimestamp = new AtomicBoolean(false); - } - - @Override - public String getSessionId() { - return sessionId; - } - - List getCookies() { - return cookies; - } - - void setCookies(List cookies) { - if ( cookies != null ) { - if(setCreatedTimestamp.compareAndSet(false, true)) { - for (ClientCookie cookie : cookies) { - // Drop the SessionId cookie received from the server. We add it every - // time we make a request with a SessionState object passed - if(cookie.getName().equalsIgnoreCase("SessionId")) continue; - // make a clone to ensure we're not holding on to any resources - // related to an HTTP connection that need to be released - this.cookies.add(new ClientCookie(cookie)); - } - created = Calendar.getInstance(); - } - } - } - - Calendar getCreatedTimestamp() { - return created; - } + + private List cookies; + private String sessionId; + private AtomicBoolean setCreatedTimestamp; + private Calendar created; + + public SessionStateImpl() { + sessionId = Long.toUnsignedString(ThreadLocalRandom.current().nextLong(), 16); + cookies = new ArrayList<>(); + setCreatedTimestamp = new AtomicBoolean(false); + } + + @Override + public String getSessionId() { + return sessionId; + } + + List getCookies() { + return cookies; + } + + void setCookies(List cookies) { + if (cookies != null) { + if (setCreatedTimestamp.compareAndSet(false, true)) { + for (ClientCookie cookie : cookies) { + // Drop the SessionId cookie received from the server. We add it every + // time we make a request with a SessionState object passed + if (cookie.getName().equalsIgnoreCase("SessionId")) continue; + this.cookies.add(cookie); + } + created = Calendar.getInstance(); + } + } + } + + Calendar getCreatedTimestamp() { + return created; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java index 60fcbbdf9..5fd30e6d0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/StreamingOutputImpl.java @@ -3,46 +3,54 @@ */ package com.marklogic.client.impl; -import java.io.IOException; -import java.io.OutputStream; - -import com.marklogic.client.util.RequestLogger; +import com.marklogic.client.impl.okhttp.RetryableRequestBody; import com.marklogic.client.io.OutputStreamSender; +import com.marklogic.client.util.RequestLogger; import okhttp3.MediaType; import okhttp3.RequestBody; import okio.BufferedSink; -class StreamingOutputImpl extends RequestBody { - private OutputStreamSender handle; - private RequestLogger logger; - private MediaType contentType; - - StreamingOutputImpl(OutputStreamSender handle, RequestLogger logger, MediaType contentType) { - super(); - this.handle = handle; - this.logger = logger; - this.contentType = contentType; - } - - @Override - public MediaType contentType() { - return contentType; - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - OutputStream out = sink.outputStream(); - - if (logger != null) { - OutputStream tee = logger.getPrintStream(); - long max = logger.getContentMax(); - if (tee != null && max > 0) { - handle.write(new OutputStreamTee(out, tee, max)); - - return; - } - } - - handle.write(out); - } +import java.io.IOException; +import java.io.OutputStream; + +class StreamingOutputImpl extends RequestBody implements RetryableRequestBody { + + private OutputStreamSender handle; + private RequestLogger logger; + private MediaType contentType; + + StreamingOutputImpl(OutputStreamSender handle, RequestLogger logger, MediaType contentType) { + super(); + this.handle = handle; + this.logger = logger; + this.contentType = contentType; + } + + @Override + public MediaType contentType() { + return contentType; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + OutputStream out = sink.outputStream(); + + if (logger != null) { + OutputStream tee = logger.getPrintStream(); + long max = logger.getContentMax(); + if (tee != null && max > 0) { + handle.write(new OutputStreamTee(out, tee, max)); + + return; + } + } + + handle.write(out); + } + + @Override + public boolean isRetryable() { + // Added in 8.0.0; streaming output cannot be retried as the stream is consumed on first write. + return false; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/TransactionImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/TransactionImpl.java index 60d25bd53..150313940 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/TransactionImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/TransactionImpl.java @@ -3,7 +3,7 @@ */ package com.marklogic.client.impl; -import com.marklogic.client.impl.ClientCookie; +import com.marklogic.client.ClientCookie; import com.marklogic.client.FailedRequestException; import com.marklogic.client.ForbiddenUserException; import com.marklogic.client.Transaction; @@ -19,7 +19,7 @@ class TransactionImpl implements Transaction { private RESTServices services; private String transactionId; private String hostId; - // we keep cookies scoped with each tranasaction to work with load balancers + // we keep cookies scoped with each transaction to work with load balancers // that need to keep requests for one transaction on a specific MarkLogic Server host private List cookies = new ArrayList<>(); private Calendar created; @@ -29,9 +29,7 @@ class TransactionImpl implements Transaction { this.transactionId = transactionId; if ( cookies != null ) { for (ClientCookie cookie : cookies) { - // make a clone to ensure we're not holding on to any resources - // related to an HTTP connection that need to be released - this.cookies.add(new ClientCookie(cookie)); + this.cookies.add(cookie); if ( "HostId".equalsIgnoreCase(cookie.getName()) ) { hostId = cookie.getValue(); } @@ -44,9 +42,6 @@ class TransactionImpl implements Transaction { public String getTransactionId() { return transactionId; } - public void setTransactionId(String transactionId) { - this.transactionId = transactionId; - } @Override public List getCookies() { @@ -57,9 +52,6 @@ public List getCookies() { public String getHostId() { return hostId; } - protected void setHostId(String hostId) { - this.hostId = hostId; - } public Calendar getCreatedTimestamp() { return created; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPKerberosAuthInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPKerberosAuthInterceptor.java similarity index 99% rename from marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPKerberosAuthInterceptor.java rename to marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPKerberosAuthInterceptor.java index 44f4de8f7..15ac1059b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPKerberosAuthInterceptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPKerberosAuthInterceptor.java @@ -1,7 +1,7 @@ /* * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.impl; +package com.marklogic.client.impl.okhttp; import java.io.IOException; import java.util.Map; @@ -20,6 +20,7 @@ import javax.security.auth.login.Configuration; import javax.security.auth.kerberos.KerberosTicket; +import com.marklogic.client.impl.SSLUtil; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPSamlAuthInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPSamlAuthInterceptor.java similarity index 97% rename from marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPSamlAuthInterceptor.java rename to marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPSamlAuthInterceptor.java index 0a58e9324..9a856306f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/HTTPSamlAuthInterceptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/HTTPSamlAuthInterceptor.java @@ -2,11 +2,12 @@ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.impl; +package com.marklogic.client.impl.okhttp; import com.marklogic.client.DatabaseClientFactory.SAMLAuthContext.AuthorizerCallback; import com.marklogic.client.DatabaseClientFactory.SAMLAuthContext.ExpiringSAMLAuth; import com.marklogic.client.DatabaseClientFactory.SAMLAuthContext.RenewerCallback; +import com.marklogic.client.impl.RESTServices; import okhttp3.Interceptor; import okhttp3.Request; import okhttp3.Response; @@ -55,7 +56,7 @@ public Response intercept(Chain chain) throws IOException { Request authenticatedRequest = chain.request().newBuilder() .header(RESTServices.HEADER_AUTHORIZATION, buildSamlHeader()) .build(); - + return chain.proceed(authenticatedRequest); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java index b14da8ada..e3f9d4bd1 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java @@ -4,14 +4,9 @@ package com.marklogic.client.impl.okhttp; import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.impl.HTTPKerberosAuthInterceptor; -import com.marklogic.client.impl.HTTPSamlAuthInterceptor; +import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; import com.marklogic.client.impl.SSLUtil; -import okhttp3.ConnectionPool; -import okhttp3.CookieJar; -import okhttp3.Dns; -import okhttp3.Interceptor; -import okhttp3.OkHttpClient; +import okhttp3.*; import javax.net.SocketFactory; import javax.net.ssl.HostnameVerifier; @@ -39,7 +34,8 @@ public abstract class OkHttpUtil { final private static ConnectionPool connectionPool = new ConnectionPool(); @SuppressWarnings({"unchecked", "deprecation"}) - public static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseClientFactory.SecurityContext securityContext) { + public static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseClientFactory.SecurityContext securityContext, + List clientConfigurators) { OkHttpClient.Builder clientBuilder = OkHttpUtil.newClientBuilder(); AuthenticationConfigurer authenticationConfigurer = null; @@ -55,9 +51,7 @@ public static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseC } else if (securityContext instanceof DatabaseClientFactory.CertificateAuthContext) { } else if (securityContext instanceof DatabaseClientFactory.SAMLAuthContext) { configureSAMLAuth((DatabaseClientFactory.SAMLAuthContext) securityContext, clientBuilder); - } else if (securityContext instanceof DatabaseClientFactory.ProgressDataCloudAuthContext || - // It's fine to refer to this deprecated class as it needs to be supported until Java Client 8. - securityContext instanceof DatabaseClientFactory.MarkLogicCloudAuthContext) { + } else if (securityContext instanceof DatabaseClientFactory.ProgressDataCloudAuthContext) { authenticationConfigurer = new ProgressDataCloudAuthenticationConfigurer(host); } else if (securityContext instanceof DatabaseClientFactory.OAuthContext) { authenticationConfigurer = new OAuthAuthenticationConfigurer(); @@ -82,6 +76,10 @@ public static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseC OkHttpUtil.configureSocketFactory(clientBuilder, sslContext, trustManager); OkHttpUtil.configureHostnameVerifier(clientBuilder, sslVerifier); + if (clientConfigurators != null) { + clientConfigurators.forEach(configurator -> configurator.configure(clientBuilder)); + } + return clientBuilder; } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java new file mode 100644 index 000000000..656e399c5 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl.okhttp; + +import okhttp3.Interceptor; +import okhttp3.Request; +import okhttp3.Response; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +/** + * Experimental interceptor added in 8.0.0 for retrying requests that fail due to connection issues. These issues are + * not handled by the application-level retry support in OkHttpServices, which only handles retries based on certain + * HTTP status codes. The main limitation of this approach is that it cannot retry a request that has a one-shot body, + * such as a streaming body. But for requests that don't have one-shot bodies, this interceptor can be helpful for + * retrying requests that fail due to temporary network issues or MarkLogic restarts. + */ +public class RetryIOExceptionInterceptor implements Interceptor { + + private final static Logger logger = org.slf4j.LoggerFactory.getLogger(RetryIOExceptionInterceptor.class); + + private final int maxRetries; + private final long initialDelayMs; + private final double backoffMultiplier; + private final long maxDelayMs; + + public RetryIOExceptionInterceptor(int maxRetries, long initialDelayMs, double backoffMultiplier, long maxDelayMs) { + this.maxRetries = maxRetries; + this.initialDelayMs = initialDelayMs; + this.backoffMultiplier = backoffMultiplier; + this.maxDelayMs = maxDelayMs; + } + + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request(); + + if (request.body() instanceof RetryableRequestBody body && !body.isRetryable()) { + return chain.proceed(request); + } + + for (int attempt = 0; attempt <= maxRetries; attempt++) { + try { + return chain.proceed(request); + } catch (IOException e) { + if (attempt == maxRetries || !isRetryableIOException(e)) { + logger.warn("Not retryable: {}; {}", e.getClass(), e.getMessage()); + throw e; + } + + long delay = calculateDelay(attempt); + logger.warn("Request to {} failed (attempt {}/{}): {}. Retrying in {}ms", + request.url(), attempt + 1, maxRetries, e.getMessage(), delay); + + sleep(delay); + } + } + + // This should never be reached due to loop logic, but is required for compilation. + throw new IllegalStateException("Unexpected end of retry loop"); + } + + private boolean isRetryableIOException(IOException e) { + return e instanceof ConnectException || + e instanceof SocketTimeoutException || + e instanceof UnknownHostException || + (e.getMessage() != null && ( + e.getMessage().contains("Failed to connect") || + e.getMessage().contains("unexpected end of stream") || + e.getMessage().contains("Connection reset") || + e.getMessage().contains("Read timed out") || + e.getMessage().contains("Broken pipe") + )); + } + + private long calculateDelay(int attempt) { + long delay = (long) (initialDelayMs * Math.pow(backoffMultiplier, attempt)); + return Math.min(delay, maxDelayMs); + } + + private void sleep(long delay) { + try { + Thread.sleep(delay); + } catch (InterruptedException ie) { + logger.warn("Ignoring InterruptedException while sleeping for retry delay: {}", ie.getMessage()); + } + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java new file mode 100644 index 000000000..ad35a07c3 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryableRequestBody.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl.okhttp; + +/** + * Interface for RequestBody implementations to signal whether they can be retried after an IOException. + * This is used by RetryIOExceptionInterceptor to determine if a failed request can be retried. + * Added in 8.0.0. + */ +public interface RetryableRequestBody { + /** + * @return false if this request body cannot be retried (e.g., because it consumes a stream that can only be + * read once); true if it can be safely retried. + */ + boolean isRetryable(); +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/util/RequestParameters.java b/marklogic-client-api/src/main/java/com/marklogic/client/util/RequestParameters.java index c82d5a34e..fbff95805 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/util/RequestParameters.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/util/RequestParameters.java @@ -3,187 +3,184 @@ */ package com.marklogic.client.util; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; - import com.marklogic.client.impl.RequestParametersImplementation; +import java.util.*; + /** * RequestParameters supports a map with a string as the key and * a list of strings as the value, which can represent parameters * of an operation including parameters transported over HTTP. */ -public class RequestParameters - extends RequestParametersImplementation - implements Map> -{ - /** - * Zero-argument constructor. - */ - public RequestParameters() { - super(); - } - - /** - * Set a parameter to a single value. - * @param name the parameter name - * @param value the value of the parameter - */ - public void put(String name, String value) { - List list = new ArrayList<>(); - list.add(value); - getMap().put(name, list); - } - /** - * Sets a parameter to a list of values. - * @param name the parameter - * @param values the list of values - */ - public void put(String name, String... values) { - getMap().put(name, Arrays.asList(values)); - } - /** - * Appends a value to the list for a parameter. - * @param name the parameter - * @param value the value to add to the list - */ - public void add(String name, String value) { - if (containsKey(name)) { - get(name).add(value); - } else { - put(name, value); - } - } - /** - * Appends a list of values to the list for a parameter. - * @param name the parameter - * @param values the values to add to the list - */ - public void add(String name, String... values) { - if (containsKey(name)) { - List list = get(name); - for (String value: values) { - list.add(value); - } - } else { - put(name, values); - } - } - - /** - * Returns the number of request parameters. - */ - @Override - public int size() { - return getMap().size(); - } - - /** - * Returns whether or not any request parameters have been specified. - */ - @Override - public boolean isEmpty() { - return getMap().isEmpty(); - } - - /** - * Checks whether the parameter name has been specified. - */ - @Override - public boolean containsKey(Object key) { - return getMap().containsKey(key); - } - - /** - * Checks whether any parameters have the value. - */ - @Override - public boolean containsValue(Object value) { - return getMap().containsValue(value); - } - - /** - * Gets the values for a parameter name. - */ - @Override - public List get(Object key) { - return getMap().get(key); - } - - /** - * Sets the values of a parameter name, returning the previous values if any. - */ - @Override - public List put(String key, List value) { - return getMap().put(key, value); - } - - /** - * Removes a parameter name, returning its values if any. - */ - @Override - public List remove(Object key) { - return getMap().remove(key); - } - - /** - * Adds existing parameter names and values. - */ - @Override - public void putAll(Map> m) { - getMap().putAll(m); - } - - /** - * Removes all parameters. - */ - @Override - public void clear() { - getMap().clear(); - } - - /** - * Returns the set of specified parameter names. - */ - @Override - public Set keySet() { - return getMap().keySet(); - } - - /** - * Returns a list of value lists. - */ - @Override - public Collection> values() { - return getMap().values(); - } - - /** - * Returns a set of parameter-list entries. - */ - @Override - public Set>> entrySet() { - return getMap().entrySet(); - } - - /** - * Creates a copy of the parameters, prepending a namespace prefix - * to each parameter name. - * @param prefix the prefix to prepend - * @return the copy of the parameters - */ - public RequestParameters copy(String prefix) { - String keyPrefix = prefix+":"; - - RequestParameters copy = new RequestParameters(); - for (Map.Entry> entry: entrySet()) { - copy.put(keyPrefix+entry.getKey(), entry.getValue()); - } - - return copy; - } +public class RequestParameters extends RequestParametersImplementation implements Map> { + + public RequestParameters() { + } + + /** + * Set a parameter to a single value. + * + * @param name the parameter name + * @param value the value of the parameter + */ + public void put(String name, String value) { + List list = new ArrayList<>(); + list.add(value); + getMap().put(name, list); + } + + /** + * Sets a parameter to a list of values. + * + * @param name the parameter + * @param values the list of values + */ + public void put(String name, String... values) { + getMap().put(name, Arrays.asList(values)); + } + + /** + * Appends a value to the list for a parameter. + * + * @param name the parameter + * @param value the value to add to the list + */ + public void add(String name, String value) { + if (containsKey(name)) { + get(name).add(value); + } else { + put(name, value); + } + } + + /** + * Appends a list of values to the list for a parameter. + * + * @param name the parameter + * @param values the values to add to the list + */ + public void add(String name, String... values) { + if (containsKey(name)) { + List list = get(name); + for (String value : values) { + list.add(value); + } + } else { + put(name, values); + } + } + + /** + * Returns the number of request parameters. + */ + @Override + public int size() { + return getMap().size(); + } + + /** + * Returns whether any request parameters have been specified. + */ + @Override + public boolean isEmpty() { + return getMap().isEmpty(); + } + + /** + * Checks whether the parameter name has been specified. + */ + @Override + public boolean containsKey(Object key) { + return getMap().containsKey(key); + } + + /** + * Checks whether any parameters have the value. + */ + @Override + public boolean containsValue(Object value) { + return getMap().containsValue(value); + } + + /** + * Gets the values for a parameter name. + */ + @Override + public List get(Object key) { + return getMap().get(key); + } + + /** + * Sets the values of a parameter name, returning the previous values if any. + */ + @Override + public List put(String key, List value) { + return getMap().put(key, value); + } + + /** + * Removes a parameter name, returning its values if any. + */ + @Override + public List remove(Object key) { + return getMap().remove(key); + } + + /** + * Adds existing parameter names and values. + */ + @Override + public void putAll(Map> m) { + getMap().putAll(m); + } + + /** + * Removes all parameters. + */ + @Override + public void clear() { + getMap().clear(); + } + + /** + * Returns the set of specified parameter names. + */ + @Override + public Set keySet() { + return getMap().keySet(); + } + + /** + * Returns a list of value lists. + */ + @Override + public Collection> values() { + return getMap().values(); + } + + /** + * Returns a set of parameter-list entries. + */ + @Override + public Set>> entrySet() { + return getMap().entrySet(); + } + + /** + * Creates a copy of the parameters, prepending a namespace prefix + * to each parameter name. + * + * @param prefix the prefix to prepend + * @return the copy of the parameters + */ + public RequestParameters copy(String prefix) { + String keyPrefix = prefix + ":"; + + RequestParameters copy = new RequestParameters(); + for (Map.Entry> entry : entrySet()) { + copy.put(keyPrefix + entry.getKey(), entry.getValue()); + } + + return copy; + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/OAuthAuthenticationConfigurerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/OAuthAuthenticationConfigurerTest.java index b6bee7c30..c8a2ddd8a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/OAuthAuthenticationConfigurerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/OAuthAuthenticationConfigurerTest.java @@ -4,20 +4,23 @@ package com.marklogic.client.impl.okhttp; import com.marklogic.client.DatabaseClientFactory; +import mockwebserver3.MockWebServer; import okhttp3.Request; -import okhttp3.mockwebserver.MockWebServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -public class OAuthAuthenticationConfigurerTest { +class OAuthAuthenticationConfigurerTest { @Test - void test() { - DatabaseClientFactory.OAuthContext authContext = new DatabaseClientFactory.OAuthContext("abc123"); - Request request = new Request.Builder().url(new MockWebServer().url("/url-doesnt-matter")).build(); + void test() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Request request = new Request.Builder().url(server.url("/url-doesnt-matter")).build(); - Request authenticatedRequest = new OAuthAuthenticationConfigurer().makeAuthenticatedRequest(request, authContext); - assertEquals("Bearer abc123", authenticatedRequest.header("Authorization")); + DatabaseClientFactory.OAuthContext authContext = new DatabaseClientFactory.OAuthContext("abc123"); + Request authenticatedRequest = new OAuthAuthenticationConfigurer().makeAuthenticatedRequest(request, authContext); + assertEquals("Bearer abc123", authenticatedRequest.header("Authorization")); + } } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java index 2f1496497..b1529fb2c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java @@ -4,10 +4,11 @@ package com.marklogic.client.impl.okhttp; import com.marklogic.client.ext.helper.LoggingObject; +import mockwebserver3.MockResponse; +import mockwebserver3.MockWebServer; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,15 +24,17 @@ * Uses OkHttp's MockWebServer to completely mock a MarkLogic instance so that we can control what response codes are * returned and processed by TokenAuthenticationInterceptor. */ -public class TokenAuthenticationInterceptorTest extends LoggingObject { +class TokenAuthenticationInterceptorTest extends LoggingObject { private MockWebServer mockWebServer; private FakeTokenGenerator fakeTokenGenerator; private OkHttpClient okHttpClient; @BeforeEach - void beforeEach() { + void beforeEach() throws IOException { mockWebServer = new MockWebServer(); + mockWebServer.start(); + fakeTokenGenerator = new FakeTokenGenerator(); ProgressDataCloudAuthenticationConfigurer.TokenAuthenticationInterceptor interceptor = @@ -43,6 +46,11 @@ void beforeEach() { okHttpClient = new OkHttpClient.Builder().addInterceptor(interceptor).build(); } + @AfterEach + void tearDown() { + mockWebServer.close(); + } + @Test void receive401() { enqueueResponseCodes(200, 200, 401, 200); @@ -110,7 +118,7 @@ void multipleThreads() throws Exception { */ private void enqueueResponseCodes(int... codes) { for (int code : codes) { - mockWebServer.enqueue(new MockResponse().setResponseCode(code)); + mockWebServer.enqueue(new MockResponse.Builder().code(code).build()); } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java index 98e43c4d8..9d447d098 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java @@ -15,296 +15,319 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.*; import com.marklogic.client.query.StructuredQueryBuilder.TemporalOperator; -import org.junit.jupiter.api.*; +import com.marklogic.mgmt.ManageClient; +import com.marklogic.mgmt.resource.temporal.TemporalCollectionLSQTManager; +import jakarta.xml.bind.DatatypeConverter; +import org.custommonkey.xmlunit.exceptions.XpathException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.w3c.dom.Document; -import jakarta.xml.bind.DatatypeConverter; import java.util.Calendar; import java.util.Random; import static org.custommonkey.xmlunit.XMLAssert.assertXpathEvaluatesTo; import static org.junit.jupiter.api.Assertions.*; -@TestMethodOrder(MethodOrderer.MethodName.class) -public class BitemporalTest { - // src/test/resources/bootstrap.xqy is run by com.marklogic.client.test.util.TestServerBootstrapper - // and sets up the "temporal-collection" and required underlying axes - // system-axis and valid-axis which have required underlying range indexes - // system-start, system-end, valid-start, and valid-end - static String temporalCollection = "temporal-collection"; - static XMLDocumentManager docMgr; - static QueryManager queryMgr; - static String uniqueBulkTerm = "temporalBulkDoc" + new Random().nextInt(10000); - static String uniqueTerm = "temporalDoc" + new Random().nextInt(10000); - static String docId = "test-" + uniqueTerm + ".xml"; - - @BeforeAll - public static void beforeClass() { - Common.connect(); - docMgr = Common.client.newXMLDocumentManager(); - queryMgr = Common.client.newQueryManager(); - } - @AfterAll - public static void afterClass() { - cleanUp(); - } - - @Test - public void a_testCreate() throws Exception { - String contents = "" + - "" + - "" + - "2014-08-19T00:00:00Z" + - "2014-08-19T00:00:01Z" + - ""; - TemporalDescriptor desc = docMgr.create(docMgr.newDocumentUriTemplate("xml"), - null, new StringHandle(contents), null, null, temporalCollection); - assertNotNull(desc); - assertNotNull(desc.getUri()); - assertTrue(desc.getUri().endsWith(".xml")); - String lastWriteTimestamp = desc.getTemporalSystemTime(); - Calendar lastWriteTime = DatatypeConverter.parseDateTime(lastWriteTimestamp); - assertNotNull(lastWriteTime); - } - - @Test - public void b_testBulk() throws Exception { - String prefix = "test_" + uniqueBulkTerm; - String doc1 = "" + - uniqueBulkTerm + " doc1" + - "" + - "" + - "2014-08-19T00:00:00Z" + - "2014-08-19T00:00:01Z" + - ""; - String doc2 = "" + - uniqueBulkTerm + " doc2" + - "" + - "" + - "2014-08-19T00:00:02Z" + - "2014-08-19T00:00:03Z" + - ""; - String doc3 = "" + - uniqueBulkTerm + " doc3" + - "" + - "" + - "2014-08-19T00:00:03Z" + - "2014-08-19T00:00:04Z" + - ""; - String doc4 = "" + - uniqueBulkTerm + " doc4" + - "" + - "" + - "2014-08-19T00:00:05Z" + - "2014-08-19T00:00:06Z" + - ""; - DocumentWriteSet writeSet = docMgr.newWriteSet(); - writeSet.add(prefix + "_1.xml", new StringHandle(doc1).withFormat(Format.XML)); - writeSet.add(prefix + "_2.xml", new StringHandle(doc2).withFormat(Format.XML)); - writeSet.add(prefix + "_3.xml", new StringHandle(doc3).withFormat(Format.XML)); - writeSet.add(prefix + "_4.xml", new StringHandle(doc4).withFormat(Format.XML)); - docMgr.write(writeSet, null, null, temporalCollection); - // do it one more time so we have two versions of each - writeSet = docMgr.newWriteSet(); - writeSet.add(prefix + "_1.xml", new StringHandle(doc1).withFormat(Format.XML)); - writeSet.add(prefix + "_2.xml", new StringHandle(doc2).withFormat(Format.XML)); - writeSet.add(prefix + "_3.xml", new StringHandle(doc3).withFormat(Format.XML)); - writeSet.add(prefix + "_4.xml", new StringHandle(doc4).withFormat(Format.XML)); - docMgr.write(writeSet, null, null, temporalCollection); - - StringQueryDefinition query = queryMgr.newStringDefinition().withCriteria(uniqueBulkTerm); - try ( DocumentPage page = docMgr.search(query, 0) ) { - assertEquals(8, page.size()); - for ( DocumentRecord record : page ) { - Document doc = record.getContentAs(Document.class); - if ( record.getUri().startsWith(prefix + "_1") ) { - assertXpathEvaluatesTo("2014-08-19T00:00:00Z", "//valid-start", doc); - continue; - } else if ( record.getUri().startsWith(prefix + "_2") ) { - assertXpathEvaluatesTo("2014-08-19T00:00:02Z", "//valid-start", doc); - continue; - } else if ( record.getUri().startsWith(prefix + "_3") ) { - assertXpathEvaluatesTo("2014-08-19T00:00:03Z", "//valid-start", doc); - continue; - } else if ( record.getUri().startsWith(prefix + "_4") ) { - assertXpathEvaluatesTo("2014-08-19T00:00:05Z", "//valid-start", doc); - continue; - } - throw new IllegalStateException("Unexpected doc:[" + record.getUri() + "]"); - } - } - } - - @Test - public void c_testOther() throws Exception { - - String version1 = "" + - uniqueTerm + " version1" + - "" + - "" + - "2014-08-19T00:00:00Z" + - "2014-08-19T00:00:01Z" + - ""; - String version2 = "" + - uniqueTerm + " version2" + - "" + - "" + - "2014-08-19T00:00:02Z" + - "2014-08-19T00:00:03Z" + - ""; - String version3 = "" + - uniqueTerm + " version3" + - "" + - "" + - "2014-08-19T00:00:03Z" + - "2014-08-19T00:00:04Z" + - ""; - String version4 = "" + - uniqueTerm + " version4" + - "" + - "" + - "2014-08-19T00:00:05Z" + - "2014-08-19T00:00:06Z" + - ""; - - // write four versions of the same document - StringHandle handle1 = new StringHandle(version1).withFormat(Format.XML); - docMgr.write(docId, null, handle1, null, null, temporalCollection); - StringHandle handle2 = new StringHandle(version2).withFormat(Format.XML); - docMgr.write(docId, null, handle2, null, null, temporalCollection); - StringHandle handle3 = new StringHandle(version3).withFormat(Format.XML); - TemporalDescriptor desc = docMgr.write(docId, null, handle3, null, null, temporalCollection); - assertNotNull(desc); - assertEquals(docId, desc.getUri()); - String thirdWriteTimestamp = desc.getTemporalSystemTime(); - assertNotNull(thirdWriteTimestamp); - - StringHandle handle4 = new StringHandle(version4).withFormat(Format.XML); - docMgr.write(docId, null, handle4, null, null, temporalCollection); - - // make sure non-temporal document read only returns the latest version - try ( DocumentPage readResults = docMgr.read(docId) ) { - assertEquals(1, readResults.size()); - DocumentRecord latestDoc = readResults.next(); - assertEquals(docId, latestDoc.getUri()); - } - - // make sure a simple term query returns all versions of bulk and other docs - StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder(); - StructuredQueryDefinition termsQuery = - sqb.or( sqb.term(uniqueTerm), sqb.term(uniqueBulkTerm) ); - long start = 1; - try ( DocumentPage termQueryResults = docMgr.search(termsQuery, start) ) { - assertEquals(12, termQueryResults.size()); - } - - StructuredQueryDefinition currentQuery = sqb.temporalLsqtQuery(temporalCollection, thirdWriteTimestamp, 1); - StructuredQueryDefinition currentDocQuery = sqb.and(termsQuery, currentQuery); - try { - // query with lsqt of last inserted document - // will throw an error because lsqt has not yet advanced - try ( DocumentPage results = docMgr.search(currentDocQuery, start) ) { - fail("Negative test should have generated a FailedRequestException of type TEMPORAL-GTLSQT"); - } - } catch (FailedRequestException e) { - assertTrue(e.getMessage().contains("TEMPORAL-GTLSQT")); - } - - // now update lsqt - Common.connectServerAdmin().newXMLDocumentManager().advanceLsqt(temporalCollection); - - // query again with lsqt of last inserted document - // will match the first three versions -- not the last because it's equal to - // not greater than the timestamp of this lsqt query - try ( DocumentPage currentDocQueryResults = docMgr.search(currentDocQuery, start) ) { - assertEquals(11, currentDocQueryResults.size()); - } - - // query with blank lsqt indicating current time - // will match all four versions - currentQuery = sqb.temporalLsqtQuery(temporalCollection, "", 1); - currentDocQuery = sqb.and(termsQuery, currentQuery); - try ( DocumentPage currentDocQueryResults = docMgr.search(currentDocQuery, start) ) { - assertEquals(12, currentDocQueryResults.size()); - } - - StructuredQueryBuilder.Axis validAxis = sqb.axis("valid-axis"); - - // create a time axis to query the versions against - Calendar start1 = DatatypeConverter.parseDateTime("2014-08-19T00:00:00Z"); - Calendar end1 = DatatypeConverter.parseDateTime("2014-08-19T00:00:04Z"); - StructuredQueryBuilder.Period period1 = sqb.period(start1, end1); - - // find all documents contained in the time range of our query axis - StructuredQueryDefinition periodQuery1 = sqb.and(termsQuery, - sqb.temporalPeriodRange(validAxis, TemporalOperator.ALN_CONTAINED_BY, period1)); - try ( DocumentPage periodQuery1Results = docMgr.search(periodQuery1, start) ) { - assertEquals(3, periodQuery1Results.size()); - } - - // create a second time axis to query the versions against - Calendar start2 = DatatypeConverter.parseDateTime("2014-08-19T00:00:04Z"); - Calendar end2 = DatatypeConverter.parseDateTime("2014-08-19T00:00:07Z"); - StructuredQueryBuilder.Period period2 = sqb.period(start2, end2); - - // find all documents contained in the time range of our second query axis - StructuredQueryDefinition periodQuery2 = sqb.and(termsQuery, - sqb.temporalPeriodRange(validAxis, TemporalOperator.ALN_CONTAINED_BY, period2)); - try ( DocumentPage periodQuery2Results = docMgr.search(periodQuery2, start) ) { - assertEquals(3, periodQuery2Results.size()); - for ( DocumentRecord result : periodQuery2Results ) { - if ( docId.equals(result.getUri()) ) { - continue; - } else if ( result.getUri().startsWith("test_" + uniqueBulkTerm + "_4") ) { - continue; - } - fail("Unexpected uri for ALN_CONTAINED_BY test:" + result.getUri()); - } - } - - // find all documents where valid time is after system time in the document - StructuredQueryBuilder.Axis systemAxis = sqb.axis("system-axis"); - StructuredQueryDefinition periodCompareQuery1 = sqb.and(termsQuery, - sqb.temporalPeriodCompare(systemAxis, TemporalOperator.ALN_AFTER, validAxis)); - try ( DocumentPage periodCompareQuery1Results = docMgr.search(periodCompareQuery1, start) ) { - assertEquals(12, periodCompareQuery1Results.size()); - } - - // find all documents where valid time is before system time in the document - StructuredQueryDefinition periodCompareQuery2 = sqb.and(termsQuery, - sqb.temporalPeriodCompare(systemAxis, TemporalOperator.ALN_BEFORE, validAxis)); - try ( DocumentPage periodCompareQuery2Results = docMgr.search(periodCompareQuery2, start) ) { - assertEquals(0, periodCompareQuery2Results.size()); - } - - // check that we get a system time when we delete - desc = docMgr.delete(docId, null, temporalCollection); - assertNotNull(desc); - assertEquals(docId, desc.getUri()); - assertNotNull(desc.getTemporalSystemTime()); - - } - - - static public void cleanUp() { - DatabaseClient client = Common.newServerAdminClient(); - try { - QueryManager queryMgr = client.newQueryManager(); - queryMgr.setPageLength(1000); - QueryDefinition query = queryMgr.newStringDefinition(); - query.setCollections(temporalCollection); - // DeleteQueryDefinition deleteQuery = client.newQueryManager().newDeleteDefinition(); - // deleteQuery.setCollections(temporalCollection); - // client.newQueryManager().delete(deleteQuery); - SearchHandle handle = queryMgr.search(query, new SearchHandle()); - MatchDocumentSummary[] docs = handle.getMatchResults(); - for ( MatchDocumentSummary doc : docs ) { - if ( ! (temporalCollection + ".lsqt").equals(doc.getUri()) ) { - client.newXMLDocumentManager().delete(doc.getUri()); - } - } - } finally { - client.release(); - } - } +class BitemporalTest { + + static String temporalCollection = "temporal-collection"; + static XMLDocumentManager docMgr; + static QueryManager queryMgr; + static String uniqueBulkTerm = "temporalBulkDoc" + new Random().nextInt(10000); + static String uniqueTerm = "temporalDoc" + new Random().nextInt(10000); + static String docId = "test-" + uniqueTerm + ".xml"; + + @BeforeEach + void setup() { + Common.connect(); + docMgr = Common.client.newXMLDocumentManager(); + queryMgr = Common.client.newQueryManager(); + } + + @AfterEach + void teardown() { + try (DatabaseClient client = Common.newServerAdminClient()) { + QueryManager queryMgr = client.newQueryManager(); + queryMgr.setPageLength(1000); + QueryDefinition query = queryMgr.newStringDefinition(); + query.setCollections(temporalCollection); + SearchHandle handle = queryMgr.search(query, new SearchHandle()); + MatchDocumentSummary[] docs = handle.getMatchResults(); + for (MatchDocumentSummary doc : docs) { + if (!(temporalCollection + ".lsqt").equals(doc.getUri())) { + client.newXMLDocumentManager().delete(doc.getUri()); + } + } + } + } + + @Test + void writeTemporalDoc() { + String contents = """ + + + + 2014-08-19T00:00:00Z + 2014-08-19T00:00:01Z + """; + + TemporalDescriptor desc = docMgr.create(docMgr.newDocumentUriTemplate("xml"), + null, new StringHandle(contents), null, null, temporalCollection); + assertNotNull(desc); + assertNotNull(desc.getUri()); + assertTrue(desc.getUri().endsWith(".xml")); + + String lastWriteTimestamp = desc.getTemporalSystemTime(); + Calendar lastWriteTime = DatatypeConverter.parseDateTime(lastWriteTimestamp); + assertNotNull(lastWriteTime); + } + + @Test + void writeTwoVersionsOfFourDocuments() throws XpathException { + String prefix = "test_" + uniqueBulkTerm; + String doc1 = """ + + %s doc1 + + + 2014-08-19T00:00:00Z + 2014-08-19T00:00:01Z + """.formatted(uniqueBulkTerm); + + String doc2 = """ + + %s doc2 + + + 2014-08-19T00:00:02Z + 2014-08-19T00:00:03Z + """.formatted(uniqueBulkTerm); + + String doc3 = """ + + %s doc3 + + + 2014-08-19T00:00:03Z + 2014-08-19T00:00:04Z + """.formatted(uniqueBulkTerm); + + String doc4 = """ + + %s doc4 + + + 2014-08-19T00:00:05Z + 2014-08-19T00:00:06Z + """.formatted(uniqueBulkTerm); + + DocumentWriteSet writeSet = docMgr.newWriteSet(); + writeSet.add(prefix + "_1.xml", new StringHandle(doc1)); + writeSet.add(prefix + "_2.xml", new StringHandle(doc2)); + writeSet.add(prefix + "_3.xml", new StringHandle(doc3)); + writeSet.add(prefix + "_4.xml", new StringHandle(doc4)); + docMgr.write(writeSet, null, null, temporalCollection); + + // do it one more time so we have two versions of each + writeSet = docMgr.newWriteSet(); + writeSet.add(prefix + "_1.xml", new StringHandle(doc1)); + writeSet.add(prefix + "_2.xml", new StringHandle(doc2)); + writeSet.add(prefix + "_3.xml", new StringHandle(doc3)); + writeSet.add(prefix + "_4.xml", new StringHandle(doc4)); + docMgr.write(writeSet, null, null, temporalCollection); + + StringQueryDefinition query = queryMgr.newStringDefinition().withCriteria(uniqueBulkTerm); + try (DocumentPage page = docMgr.search(query, 0)) { + assertEquals(8, page.size()); + for (DocumentRecord record : page) { + Document doc = record.getContentAs(Document.class); + if (record.getUri().startsWith(prefix + "_1")) { + assertXpathEvaluatesTo("2014-08-19T00:00:00Z", "//valid-start", doc); + continue; + } else if (record.getUri().startsWith(prefix + "_2")) { + assertXpathEvaluatesTo("2014-08-19T00:00:02Z", "//valid-start", doc); + continue; + } else if (record.getUri().startsWith(prefix + "_3")) { + assertXpathEvaluatesTo("2014-08-19T00:00:03Z", "//valid-start", doc); + continue; + } else if (record.getUri().startsWith(prefix + "_4")) { + assertXpathEvaluatesTo("2014-08-19T00:00:05Z", "//valid-start", doc); + continue; + } + throw new IllegalStateException("Unexpected doc:[" + record.getUri() + "]"); + } + } + } + + @Test + void lsqtTest() { + // Due to bug MLE-24511 where LSQT properties aren't updated correctly in ml-gradle 6.0.0, we need to manually + // deploy them for this test. + ManageClient manageClient = Common.newManageClient(); + TemporalCollectionLSQTManager mgr = new TemporalCollectionLSQTManager(manageClient, "java-unittest", "temporal-collection"); + String payload = """ + { + "lsqt-enabled": true, + "automation": { + "enabled": true, + "period": 5000 + } + } + """; + mgr.save(payload); + + String version1 = """ + + %s version1 + + + 2014-08-19T00:00:00Z + 2014-08-19T00:00:01Z + """.formatted(uniqueTerm); + + String version2 = """ + + %s version2 + + + 2014-08-19T00:00:02Z + 2014-08-19T00:00:03Z + """.formatted(uniqueTerm); + + String version3 = """ + + %s version3 + + + 2014-08-19T00:00:03Z + 2014-08-19T00:00:04Z + """.formatted(uniqueTerm); + + String version4 = """ + + %s version4 + + + 2014-08-19T00:00:05Z + 2014-08-19T00:00:06Z + """.formatted(uniqueTerm); + + // write four versions of the same document + StringHandle handle1 = new StringHandle(version1).withFormat(Format.XML); + docMgr.write(docId, null, handle1, null, null, temporalCollection); + StringHandle handle2 = new StringHandle(version2).withFormat(Format.XML); + docMgr.write(docId, null, handle2, null, null, temporalCollection); + StringHandle handle3 = new StringHandle(version3).withFormat(Format.XML); + TemporalDescriptor desc = docMgr.write(docId, null, handle3, null, null, temporalCollection); + + assertNotNull(desc); + assertEquals(docId, desc.getUri()); + String thirdWriteTimestamp = desc.getTemporalSystemTime(); + assertNotNull(thirdWriteTimestamp); + + StringHandle handle4 = new StringHandle(version4).withFormat(Format.XML); + docMgr.write(docId, null, handle4, null, null, temporalCollection); + + // make sure non-temporal document read only returns the latest version + try (DocumentPage readResults = docMgr.read(docId)) { + assertEquals(1, readResults.size()); + DocumentRecord latestDoc = readResults.next(); + assertEquals(docId, latestDoc.getUri()); + } + + // make sure a simple term query returns all versions of bulk and other docs + StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder(); + StructuredQueryDefinition termsQuery = + sqb.or(sqb.term(uniqueTerm), sqb.term(uniqueBulkTerm)); + long start = 1; + try (DocumentPage termQueryResults = docMgr.search(termsQuery, start)) { + assertEquals(4, termQueryResults.size()); + } + + StructuredQueryDefinition currentQuery = sqb.temporalLsqtQuery(temporalCollection, thirdWriteTimestamp, 1); + StructuredQueryDefinition currentDocQuery = sqb.and(termsQuery, currentQuery); + + final StructuredQueryDefinition queryThatWillFail = currentDocQuery; + FailedRequestException ex = assertThrows(FailedRequestException.class, () -> docMgr.search(queryThatWillFail, start)); + String message = ex.getMessage(); + assertTrue(message.contains("TEMPORAL"), "The query should fail, but the actual error code " + + "depends on the MarkLogic version. Prior to 12.1, the code was TEMPORAL-GTLSQT. " + + "On the develop branch for 12.1, it's TEMPORAL-NOLSQT. Actual message: " + message); + + try (DatabaseClient client = Common.newServerAdminClient()) { + client.newXMLDocumentManager().advanceLsqt(temporalCollection); + } + + // query again with lsqt of last inserted document + // will match the first three versions -- not the last because it's equal to + // not greater than the timestamp of this lsqt query + try (DocumentPage currentDocQueryResults = docMgr.search(currentDocQuery, start)) { + assertEquals(3, currentDocQueryResults.size()); + } + + // query with blank lsqt indicating current time + // will match all four versions + currentQuery = sqb.temporalLsqtQuery(temporalCollection, "", 1); + currentDocQuery = sqb.and(termsQuery, currentQuery); + try (DocumentPage currentDocQueryResults = docMgr.search(currentDocQuery, start)) { + assertEquals(4, currentDocQueryResults.size()); + } + + StructuredQueryBuilder.Axis validAxis = sqb.axis("valid-axis"); + + // create a time axis to query the versions against + Calendar start1 = DatatypeConverter.parseDateTime("2014-08-19T00:00:00Z"); + Calendar end1 = DatatypeConverter.parseDateTime("2014-08-19T00:00:04Z"); + StructuredQueryBuilder.Period period1 = sqb.period(start1, end1); + + // find all documents contained in the time range of our query axis + StructuredQueryDefinition periodQuery1 = sqb.and(termsQuery, + sqb.temporalPeriodRange(validAxis, TemporalOperator.ALN_CONTAINED_BY, period1)); + try (DocumentPage periodQuery1Results = docMgr.search(periodQuery1, start)) { + assertEquals(1, periodQuery1Results.size()); + } + + // create a second time axis to query the versions against + Calendar start2 = DatatypeConverter.parseDateTime("2014-08-19T00:00:04Z"); + Calendar end2 = DatatypeConverter.parseDateTime("2014-08-19T00:00:07Z"); + StructuredQueryBuilder.Period period2 = sqb.period(start2, end2); + + // find all documents contained in the time range of our second query axis + StructuredQueryDefinition periodQuery2 = sqb.and(termsQuery, + sqb.temporalPeriodRange(validAxis, TemporalOperator.ALN_CONTAINED_BY, period2)); + try (DocumentPage periodQuery2Results = docMgr.search(periodQuery2, start)) { + assertEquals(1, periodQuery2Results.size()); + for (DocumentRecord result : periodQuery2Results) { + if (docId.equals(result.getUri())) { + continue; + } else if (result.getUri().startsWith("test_" + uniqueBulkTerm + "_4")) { + continue; + } + fail("Unexpected uri for ALN_CONTAINED_BY test:" + result.getUri()); + } + } + + // find all documents where valid time is after system time in the document + StructuredQueryBuilder.Axis systemAxis = sqb.axis("system-axis"); + StructuredQueryDefinition periodCompareQuery1 = sqb.and(termsQuery, + sqb.temporalPeriodCompare(systemAxis, TemporalOperator.ALN_AFTER, validAxis)); + try (DocumentPage periodCompareQuery1Results = docMgr.search(periodCompareQuery1, start)) { + assertEquals(4, periodCompareQuery1Results.size()); + } + + // find all documents where valid time is before system time in the document + StructuredQueryDefinition periodCompareQuery2 = sqb.and(termsQuery, + sqb.temporalPeriodCompare(systemAxis, TemporalOperator.ALN_BEFORE, validAxis)); + try (DocumentPage periodCompareQuery2Results = docMgr.search(periodCompareQuery2, start)) { + assertEquals(0, periodCompareQuery2Results.size()); + } + + // check that we get a system time when we delete + desc = docMgr.delete(docId, null, temporalCollection); + assertNotNull(desc); + assertEquals(docId, desc.getUri()); + assertNotNull(desc.getTemporalSystemTime()); + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java index bf2d51a47..2437bf255 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java @@ -8,6 +8,8 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientBuilder; import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; +import com.marklogic.client.impl.okhttp.RetryIOExceptionInterceptor; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.ManageConfig; @@ -29,6 +31,12 @@ public class Common { + static { + DatabaseClientFactory.removeConfigurators(); + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) client -> + client.addInterceptor(new RetryIOExceptionInterceptor(3, 1000, 2, 8000))); + } + final public static String USER = "rest-writer"; final public static String PASS = "x"; final public static String REST_ADMIN_USER = "rest-admin"; @@ -68,7 +76,6 @@ public X509Certificate[] getAcceptedIssuers() { public static DatabaseClient client; public static DatabaseClient restAdminClient; - public static DatabaseClient serverAdminClient; public static DatabaseClient evalClient; public static DatabaseClient readOnlyClient; @@ -84,12 +91,6 @@ public static DatabaseClient connectRestAdmin() { return restAdminClient; } - public static DatabaseClient connectServerAdmin() { - if (serverAdminClient == null) - serverAdminClient = newServerAdminClient(); - return serverAdminClient; - } - public static DatabaseClient connectEval() { if (evalClient == null) evalClient = newEvalClient(); @@ -265,9 +266,11 @@ public static ObjectNode newServerPayload() { } public static void deleteUrisWithPattern(String pattern) { - Common.connectServerAdmin().newServerEval() - .xquery(String.format("cts:uri-match('%s') ! xdmp:document-delete(.)", pattern)) - .evalAs(String.class); + try (DatabaseClient client = Common.newServerAdminClient()) { + client.newServerEval() + .xquery(String.format("cts:uri-match('%s') ! xdmp:document-delete(.)", pattern)) + .evalAs(String.class); + } } /** diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/MarkLogicCloudAuthenticationDebugger.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ProgressDataCloudAuthenticationDebugger.java similarity index 97% rename from marklogic-client-api/src/test/java/com/marklogic/client/test/MarkLogicCloudAuthenticationDebugger.java rename to marklogic-client-api/src/test/java/com/marklogic/client/test/ProgressDataCloudAuthenticationDebugger.java index b4d3c7611..e542317b6 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/MarkLogicCloudAuthenticationDebugger.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ProgressDataCloudAuthenticationDebugger.java @@ -16,7 +16,7 @@ * "localhost" as the cloud host, "username:password" (often "admin:the admin password") as the apiKey, and * "local/manage" as the basePath. */ -public class MarkLogicCloudAuthenticationDebugger { +public class ProgressDataCloudAuthenticationDebugger { public static void main(String[] args) throws Exception { String cloudHost = args[0]; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/RowBatcherTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/RowBatcherTest.java index 91e4cc293..5caa9ba2a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/RowBatcherTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/RowBatcherTest.java @@ -24,6 +24,7 @@ import com.marklogic.client.type.PlanSystemColumn; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -39,7 +40,8 @@ import static org.junit.jupiter.api.Assertions.*; -public class RowBatcherTest { +class RowBatcherTest { + private final static String TEST_DIR = "/test/rowbatch/unit/"; private final static String TEST_COLLECTION = TEST_DIR+"codes"; private final static String TABLE_NS_URI = "http://marklogic.com/table"; @@ -190,6 +192,8 @@ public void testJsonDocs1Thread() throws Exception { } @Test + @Disabled("Disabled due to https://progresssoftware.atlassian.net/browse/MLE-24579 , which causes the server to restart, " + + "which can cause many other tests to fail.") void noRowsReturned() { RowBatcher rowBatcher = jsonBatcher(1); RowManager rowMgr = rowBatcher.getRowManager(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchWithOptionsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchWithOptionsTest.java index aa07928b9..acf3651e3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchWithOptionsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchWithOptionsTest.java @@ -8,7 +8,6 @@ import com.marklogic.client.row.RowRecord; import com.marklogic.client.test.junit5.RequiresML12; import com.marklogic.client.type.PlanSearchOptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,15 +44,14 @@ void badBm25LengthWeight() { } @Test - @Disabled("zero and random aren't in the 12 EA release.") void zero() { -// rowManager.withUpdate(false); -// PlanSearchOptions options = op.searchOptions().withScoreMethod(PlanSearchOptions.ScoreMethod.ZERO); -// List rows = resultRows(op.fromSearch(op.cts.wordQuery("saxophone"), null, null, options)); -// assertEquals(2, rows.size()); -// rows.forEach(row -> { -// assertEquals(0, row.getInt("score"), "The score for every row should be 0."); -// }); + rowManager.withUpdate(false); + PlanSearchOptions options = op.searchOptions().withScoreMethod(PlanSearchOptions.ScoreMethod.ZERO); + List rows = resultRows(op.fromSearch(op.cts.wordQuery("saxophone"), null, null, options)); + assertEquals(2, rows.size()); + rows.forEach(row -> { + assertEquals(0, row.getInt("score"), "The score for every row should be 0."); + }); } @Test diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java index 44915e845..4118b5d7a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java @@ -20,8 +20,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledOnJre; -import org.junit.jupiter.api.condition.JRE; import org.junit.jupiter.api.extension.ExtendWith; import javax.net.ssl.SSLContext; @@ -162,9 +160,6 @@ void tLS13ClientWithTLS12Server() { } @ExtendWith(RequiresML12.class) - // The TLSv1.3 tests are failing on Java 8, because TLSv1.3 is disabled with our version of Java 8. - // There may be a way to configure Java 8 to use TLSv1.3, but it is not currently working. - @DisabledOnJre(JRE.JAVA_8) @Test void tLS13ClientWithTLS13Server() { setAppServerMinimumTLSVersion("TLSv1.3"); @@ -177,7 +172,6 @@ void tLS13ClientWithTLS13Server() { } @ExtendWith(RequiresML12.class) - @DisabledOnJre(JRE.JAVA_8) @Test void tLS12ClientWithTLS13ServerShouldFail() { setAppServerMinimumTLSVersion("TLSv1.3"); diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index 84c83e245..f7bfb277f 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -2,6 +2,8 @@ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + plugins { id "groovy" id 'maven-publish' @@ -12,9 +14,19 @@ plugins { dependencies { compileOnly gradleApi() - implementation project(':marklogic-client-api') - implementation 'org.jetbrains.kotlin:kotlin-stdlib:2.1.0' + + // This is a runtime dependency of marklogic-client-api but is needed for compiling. + compileOnly "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + + // Gradle 9 does not like for a plugin to have a project dependency; trying to publish it results in a + // NoSuchMethodError pertaining to getProjectDependency. So treating this as a 3rd party dependency. This creates + // additional work during development, though we rarely modify the code in this plugin anymore. + implementation "com.marklogic:marklogic-client-api:${version}" + + implementation 'org.jetbrains.kotlin:kotlin-stdlib:2.2.20' implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${jacksonVersion}" + + // Sticking with this older version for now as the latest 1.x version introduces breaking changes. implementation 'com.networknt:json-schema-validator:1.0.88' // Not yet migrating this project to JUnit 5. Will reconsider it once we have a reason to enhance @@ -23,13 +35,13 @@ dependencies { testImplementation 'xmlunit:xmlunit:1.6' testCompileOnly gradleTestKit() - testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' + testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } // Added to avoid problem where processResources fails because - somehow - the plugin properties file is getting // copied twice. This started occurring with the upgrade of Gradle from 6.x to 7.x. tasks.processResources { - duplicatesStrategy = "exclude" + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } tasks.register("mlDevelopmentToolsJar", Jar) { @@ -45,18 +57,13 @@ gradlePlugin { id = 'com.marklogic.ml-development-tools' displayName = 'ml-development-tools MarkLogic Data Service Tools' description = 'ml-development-tools plugin for developing data services on MarkLogic' - tags.set(['marklogic', 'progress']) + tags = ['marklogic', 'progress'] implementationClass = 'com.marklogic.client.tools.gradle.ToolsPlugin' } } } publishing { - publications { - main(MavenPublication) { - from components.java - } - } repositories { maven { if (project.hasProperty("mavenUser")) { @@ -70,11 +77,10 @@ publishing { } } -compileKotlin { - kotlinOptions.jvmTarget = '1.8' -} -compileTestKotlin { - kotlinOptions.jvmTarget = '1.8' +tasks.withType(KotlinCompile).configureEach { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } } tasks.register("generateTests", JavaExec) { diff --git a/ml-development-tools/src/test/example-project/build.gradle b/ml-development-tools/src/test/example-project/build.gradle index c0ce194f3..0ec53c470 100644 --- a/ml-development-tools/src/test/example-project/build.gradle +++ b/ml-development-tools/src/test/example-project/build.gradle @@ -4,7 +4,7 @@ buildscript { mavenCentral() } dependencies { - classpath "com.marklogic:ml-development-tools:7.2.0" + classpath "com.marklogic:ml-development-tools:8.0.0" } } @@ -23,11 +23,11 @@ repositories { } dependencies { - implementation 'com.marklogic:marklogic-client-api:7.2.0' + implementation 'com.marklogic:marklogic-client-api:8.0.0' } tasks.register("testFullPath", com.marklogic.client.tools.gradle.EndpointProxiesGenTask) { - serviceDeclarationFile = "/Users/rrudin/workspace/java-client-api/example-project/src/main/ml-modules/root/inventory/service.json" + serviceDeclarationFile = "/Users/rudin/workspace/java-client-api/ml-development-tools/src/test/example-project/src/main/ml-modules/root/inventory/service.json" } tasks.register("testProjectPath", com.marklogic.client.tools.gradle.EndpointProxiesGenTask) { diff --git a/pom.xml b/pom.xml index 210ca83e1..71741b24c 100644 --- a/pom.xml +++ b/pom.xml @@ -11,53 +11,42 @@ It is not intended to be used to build this project. 4.0.0 com.marklogic marklogic-client-api - 7.2.0 + 8.0.0 jakarta.xml.bind jaxb-api - 3.0.1 - - - org.glassfish.jaxb - jaxb-runtime - 3.0.2 + 4.0.4 runtime org.glassfish.jaxb - jaxb-core - 3.0.2 + jaxb-runtime + 4.0.6 runtime com.squareup.okhttp3 okhttp - 4.12.0 + 5.2.0 runtime com.squareup.okhttp3 logging-interceptor - 4.12.0 + 5.2.0 runtime io.github.rburgst okhttp-digest - 2.7 + 3.1.1 runtime com.sun.mail jakarta.mail - 2.0.1 - runtime - - - javax.ws.rs - javax.ws.rs-api - 2.1.1 + 2.0.2 runtime @@ -69,13 +58,13 @@ It is not intended to be used to build this project. com.fasterxml.jackson.core jackson-databind - 2.19.0 + 2.20.0 runtime com.fasterxml.jackson.dataformat jackson-dataformat-csv - 2.19.0 + 2.20.0 runtime diff --git a/test-app/README.md b/test-app/README.md index f397f7b9f..6db548bb7 100644 --- a/test-app/README.md +++ b/test-app/README.md @@ -34,3 +34,8 @@ You can also specify custom mappings via the Gradle task. For example, if you ha port 8123 and you want to associate a path of "/my/custom/server" to it, you can do: ./gradlew runBlock -PrpsCustomMappings=/my/custom/server,8123 + +To run one or more tests with the reverse proxy server being started, the tests being run, and then the server being +stopped, do the following (you can see examples of this in the project `Jenkinsfile` as well): + + ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test --tests ReadDocumentPageTest diff --git a/test-app/build.gradle b/test-app/build.gradle index 86cd75b11..a06a500b5 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -1,16 +1,20 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ + plugins { - id 'com.marklogic.ml-gradle' version '5.0.0' - id 'java' + id "net.saliman.properties" version "1.5.2" + id 'com.marklogic.ml-gradle' version '6.0.1' id "com.github.psxpaul.execfork" version "0.2.2" } dependencies { - implementation "io.undertow:undertow-core:2.2.37.Final" - implementation "io.undertow:undertow-servlet:2.2.37.Final" + implementation "io.undertow:undertow-core:2.3.19.Final" + implementation "io.undertow:undertow-servlet:2.3.19.Final" implementation 'org.slf4j:slf4j-api:2.0.17' - implementation 'ch.qos.logback:logback-classic:1.3.15' + implementation 'ch.qos.logback:logback-classic:1.5.18' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" - implementation 'com.squareup.okhttp3:okhttp:4.12.0' + implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } // See https://github.com/psxpaul/gradle-execfork-plugin for docs. @@ -22,9 +26,9 @@ tasks.register("runReverseProxyServer", com.github.psxpaul.task.JavaExecFork) { "directly to MarkLogic" classpath = sourceSets.main.runtimeClasspath main = "com.marklogic.client.test.ReverseProxyServer" - workingDir = "$buildDir" - standardOutput = file("$buildDir/reverse-proxy.log") - errorOutput = file("$buildDir/reverse-proxy-error.log") + workingDir = "${layout.buildDirectory.get()}" + standardOutput = file("${layout.buildDirectory.get()}/reverse-proxy.log") + errorOutput = file("${layout.buildDirectory.get()}/reverse-proxy-error.log") } tasks.register("runBlockingReverseProxyServer", JavaExec) {