From 0086481cd9567af7747a82ea24c852156eca842a Mon Sep 17 00:00:00 2001 From: Vladimir Sitnikov Date: Wed, 12 Feb 2025 16:46:17 +0300 Subject: [PATCH] Add CassandraDependenciesDockerJobTest to verify container-based docker job execution (#150) * Add CassandraDependenciesDockerJobTest to verify container-based docker job execution Signed-off-by: Vladimir Sitnikov * Keep classes from Cassandra Java Driver This resolves Caused by: java.lang.IllegalArgumentException: Can't find class DefaultMetricsFactory (specified by advanced.metrics.factory.class) at com.datastax.oss.driver.internal.core.util.Reflection.resolveClass(Reflection.java:302) at com.datastax.oss.driver.internal.core.util.Reflection.buildFromConfig(Reflection.java:235) at com.datastax.oss.driver.internal.core.util.Reflection.buildFromConfig(Reflection.java:110) at com.datastax.oss.driver.internal.core.context.DefaultDriverContext.buildMetricsFactory(DefaultDriverContext.java:548) Signed-off-by: Vladimir Sitnikov * Skip Elasticsearch in Cassandra CI job Signed-off-by: Vladimir Sitnikov * Group cassandra dependencies job logs in GitHub output Signed-off-by: Vladimir Sitnikov --------- Signed-off-by: Vladimir Sitnikov --- .github/workflows/ci.yaml | 15 ++++-- Dockerfile | 2 +- .../CassandraDependenciesDockerJobTest.java | 54 +++++++++++++++++++ .../CassandraDependenciesJobTest.java | 47 +++++++++------- .../cassandra/LogToConsolePrinter.java | 35 ++++++++++++ jaeger-spark-dependencies/pom.xml | 7 +++ pom.xml | 7 +++ 7 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesDockerJobTest.java create mode 100644 jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/LogToConsolePrinter.java diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 61b5a9e..205ddbd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,9 +19,18 @@ jobs: # TODO remove once testcontainers are updated - name: pull ryuk image run: docker pull testcontainersofficial/ryuk:0.3.0 + - name: Build docker image + uses: docker/build-push-action@v2.7.0 + # We need the image for the tests + with: + context: . + push: false + tags: ghcr.io/jaegertracing/spark-dependencies/spark-dependencies + - name: compile + run: ./mvnw --batch-mode clean install -DskipTests - name: test # pin Jaeger version temporarily to work around build failures - run: JAEGER_VERSION=1.52.0 ./mvnw --batch-mode clean test + run: JAEGER_VERSION=1.52.0 ./mvnw --batch-mode test -pl jaeger-spark-dependencies-cassandra latest-jaeger-es7: name: Latest Jaeger and ES7 @@ -38,7 +47,7 @@ jobs: - name: compile run: ./mvnw --batch-mode clean install -DskipTests - name: test - run: ELASTICSEARCH_VERSION=7.3.0 ./mvnw --batch-mode clean test -pl jaeger-spark-dependencies-elasticsearch + run: ELASTICSEARCH_VERSION=7.3.0 ./mvnw --batch-mode test -pl jaeger-spark-dependencies-elasticsearch latest-jaeger-es8: name: Latest Jaeger and ES8 @@ -55,4 +64,4 @@ jobs: - name: compile run: ./mvnw --batch-mode clean install -DskipTests - name: test - run: ELASTICSEARCH_VERSION=8.3.1 ./mvnw --batch-mode clean test -pl jaeger-spark-dependencies-elasticsearch + run: ELASTICSEARCH_VERSION=8.3.1 ./mvnw --batch-mode test -pl jaeger-spark-dependencies-elasticsearch diff --git a/Dockerfile b/Dockerfile index 6a9a53e..9959642 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ COPY .mvn $APP_HOME/.mvn COPY mvnw $APP_HOME WORKDIR $APP_HOME -RUN --mount=type=cache,target=/root/.m2 ./mvnw package -Dlicense.skip=true -DskipTests +RUN --mount=type=cache,target=/root/.m2 ./mvnw package --batch-mode -Dlicense.skip=true -DskipTests FROM eclipse-temurin:11-jre MAINTAINER Pavol Loffay diff --git a/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesDockerJobTest.java b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesDockerJobTest.java new file mode 100644 index 0000000..9a0485d --- /dev/null +++ b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesDockerJobTest.java @@ -0,0 +1,54 @@ +/** + * Copyright 2017 The Jaeger Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package io.jaegertracing.spark.dependencies.cassandra; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +public class CassandraDependenciesDockerJobTest extends CassandraDependenciesJobTest { + private static String dependenciesJobTag() { + String tag = System.getenv("SPARK_DEPENDENCIES_JOB_TAG"); + if (tag == null || tag.isEmpty()) { + return "latest"; + } + return tag.trim(); + } + + @Override + protected void deriveDependencies() { + System.out.println("::group::🚧 🚧 🚧 CassandraDependenciesDockerJob logs"); + try (GenericContainer sparkDependenciesJob = new GenericContainer<>( + DockerImageName.parse("ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:" + dependenciesJobTag())) + .withNetwork(network) + .withLogConsumer(new LogToConsolePrinter("[spark-dependencies] ")) + .withEnv("STORAGE", "cassandra") + .withEnv("CASSANDRA_KEYSPACE", "jaeger_v1_dc1") + .withEnv("CASSANDRA_CONTACT_POINTS", "cassandra") // This should be an address within the docker network + .withEnv("CASSANDRA_LOCAL_DC", cassandra.getLocalDatacenter()) + .withEnv("CASSANDRA_USERNAME", cassandra.getUsername()) + .withEnv("CASSANDRA_PASSWORD", cassandra.getPassword()) + .dependsOn(cassandra, jaegerCassandraSchema);){ + sparkDependenciesJob.start(); + await("spark-dependencies-job execution") + .atMost(3, TimeUnit.MINUTES) + .until(() -> !sparkDependenciesJob.isRunning()); + } finally { + System.out.println("::endgroup::"); + } + } +} diff --git a/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJobTest.java b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJobTest.java index 2f583bd..cb8818f 100644 --- a/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJobTest.java +++ b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJobTest.java @@ -15,8 +15,7 @@ import static org.awaitility.Awaitility.await; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Session; +import com.datastax.oss.driver.api.core.CqlSession; import io.jaegertracing.spark.dependencies.test.DependenciesTest; import java.time.LocalDate; import java.util.Collections; @@ -26,7 +25,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.testcontainers.containers.CassandraContainer; +import org.testcontainers.cassandra.CassandraContainer; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -36,17 +35,17 @@ */ public class CassandraDependenciesJobTest extends DependenciesTest { - private static Network network; - private static CassandraContainer cassandra; - private static GenericContainer jaegerCollector; - private static GenericContainer jaegerQuery; - private static GenericContainer jaegerCassandraSchema; + protected static Network network; + protected static CassandraContainer cassandra; + protected static GenericContainer jaegerCollector; + protected static GenericContainer jaegerQuery; + protected static GenericContainer jaegerCassandraSchema; private static int cassandraPort; @BeforeClass public static void beforeClass() { network = Network.newNetwork(); - cassandra = new CassandraContainer<>("cassandra:4.1") + cassandra = new CassandraContainer("cassandra:4.1") .withNetwork(network) .withNetworkAliases("cassandra") .withExposedPorts(9042); @@ -54,7 +53,7 @@ public static void beforeClass() { cassandraPort = cassandra.getMappedPort(9042); jaegerCassandraSchema = new GenericContainer<>("jaegertracing/jaeger-cassandra-schema:" + jaegerVersion()) - .withLogConsumer(frame -> System.out.print(frame.getUtf8String())) + .withLogConsumer(new LogToConsolePrinter("[jaeger-cassandra-schema] ")) .withNetwork(network); jaegerCassandraSchema.start(); /** @@ -97,13 +96,16 @@ public static void afterClass() { @After public void after() { - try (Cluster cluster = cassandra.getCluster(); Session session = cluster.newSession()) { + try (CqlSession session = CqlSession.builder() + .addContactPoint(cassandra.getContactPoint()) + .withLocalDatacenter(cassandra.getLocalDatacenter()) + .build()) { session.execute("TRUNCATE jaeger_v1_dc1.traces"); session.execute(String.format("TRUNCATE jaeger_v1_dc1.%s", dependenciesTable(session))); } } - private String dependenciesTable(Session session) { + private String dependenciesTable(CqlSession session) { try { session.execute("SELECT ts from jaeger_v1_dc1.dependencies_v2 limit 1;"); } catch (Exception ex) { @@ -114,14 +116,19 @@ private String dependenciesTable(Session session) { @Override protected void deriveDependencies() { - CassandraDependenciesJob.builder() - .contactPoints("localhost:" + cassandraPort) - .day(LocalDate.now()) - .keyspace("jaeger_v1_dc1") - .username(cassandra.getUsername()) - .password(cassandra.getPassword()) - .build() - .run("peer.service"); + System.out.println("::group::🚧 🚧 🚧 CassandraDependenciesJob logs"); + try { + CassandraDependenciesJob.builder() + .contactPoints("localhost:" + cassandraPort) + .day(LocalDate.now()) + .keyspace("jaeger_v1_dc1") + .username(cassandra.getUsername()) + .password(cassandra.getPassword()) + .build() + .run("peer.service"); + } finally { + System.out.println("::endgroup::"); + } } @Override diff --git a/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/LogToConsolePrinter.java b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/LogToConsolePrinter.java new file mode 100644 index 0000000..0b962c7 --- /dev/null +++ b/jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/LogToConsolePrinter.java @@ -0,0 +1,35 @@ +/** + * Copyright 2017 The Jaeger Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package io.jaegertracing.spark.dependencies.cassandra; + +import org.testcontainers.containers.output.OutputFrame; + +import java.util.function.Consumer; + +public final class LogToConsolePrinter implements Consumer { + private final String prefix; + + public LogToConsolePrinter(String prefix) { + this.prefix = prefix; + } + + @Override + public void accept(OutputFrame outputFrame) { + String message = outputFrame.getUtf8String(); + if (!message.isEmpty()) { + System.out.print(prefix); + System.out.print(message); + } + } +} diff --git a/jaeger-spark-dependencies/pom.xml b/jaeger-spark-dependencies/pom.xml index f8970a5..333a889 100644 --- a/jaeger-spark-dependencies/pom.xml +++ b/jaeger-spark-dependencies/pom.xml @@ -67,6 +67,13 @@ ** + + + com.datastax.oss:* + + ** + + log4j:log4j diff --git a/pom.xml b/pom.xml index 4204a17..1a11342 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,13 @@ org.testcontainers cassandra ${version.org.testcontainers} + + + + com.datastax.cassandra + cassandra-driver-core + + com.google.guava