From 59c2422478856aaee92c3ecc7b9434f082e8483f Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 11 Apr 2024 17:38:28 -0700
Subject: [PATCH 01/29] poc, take 1
---
pom.xml | 1 +
pulsar-tracing/pom.xml | 82 +++
.../oss/pulsar/tracing/BrokerTracing.java | 645 ++++++++++++++++++
.../META-INF/services/broker_interceptor.yml | 3 +
pulsar-tracing/src/test/resources/log4j2.xml | 41 ++
5 files changed, 772 insertions(+)
create mode 100644 pulsar-tracing/pom.xml
create mode 100644 pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
create mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
create mode 100644 pulsar-tracing/src/test/resources/log4j2.xml
diff --git a/pom.xml b/pom.xml
index 0e83aa9a..8f9c2802 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,7 @@
<modules>
<module>activemq-filters</module>
<module>pulsar-jms-filters</module>
+ <module>pulsar-tracing</module>
<module>pulsar-jms-admin-api</module>
<module>pulsar-jms</module>
<module>resource-adapter</module>
diff --git a/pulsar-tracing/pom.xml b/pulsar-tracing/pom.xml
new file mode 100644
index 00000000..e4a65161
--- /dev/null
+++ b/pulsar-tracing/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Copyright DataStax, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>pulsar-jms-parent</artifactId>
+ <groupId>com.datastax.oss</groupId>
+ <version>4.1.1-alpha-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>pulsar-tracing</artifactId>
+ <packaging>jar</packaging>
+ <name>DataStax Starlight for JMS - Broker Side Filters</name>
+ <properties>
+ <build.dir>${project.build.directory}</build.dir>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${pulsar.groupId}</groupId>
+ <artifactId>pulsar-broker</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-nar-maven-plugin</artifactId>
+ <version>1.3.2</version>
+ <extensions>true</extensions>
+ <configuration>
+ <finalName>pulsar-tracing-${project.version}</finalName>
+ <classifier>nar</classifier>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-nar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>nar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
new file mode 100644
index 00000000..e18e2cec
--- /dev/null
+++ b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
@@ -0,0 +1,645 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastax.oss.pulsar.tracing;
+
+import io.netty.buffer.ByteBuf;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.intercept.BrokerInterceptor;
+import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.broker.service.Producer;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.Subscription;
+import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.BaseCommand;
+import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.naming.TopicName;
+
+@Slf4j
+public class BrokerTracing implements BrokerInterceptor {
+
+ static final int MAX_DATA_LENGTH = 1024;
+
+ public enum EventReasons {
+ ADMINISTRATIVE,
+ MESSAGE,
+ TRANSACTION,
+ SERVLET,
+ }
+
+ public enum TraceLevel {
+ NONE,
+ MIMIMAL,
+ BASIC,
+ FULL
+ }
+
+ // private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
+ private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+
+ public void initialize(PulsarService pulsarService) {
+ // defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
+ // defaultEnabledEvents.add(EventReasons.MESSAGE);
+ // defaultEnabledEvents.add(EventReasons.TRANSACTION);
+ }
+
+ @Override
+ public void close() {}
+
+ private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
+ return getEnabledEvents(cnx.getBrokerService().getPulsar());
+ }
+
+ private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
+ // todo: do this less frequently
+
+ String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", "");
+
+ Set<EventReasons> enabledEvents = new HashSet<>();
+
+ for (String event : events.split(",")) {
+ try {
+ enabledEvents.add(EventReasons.valueOf(event.trim()));
+ } catch (IllegalArgumentException e) {
+ log.warn("Invalid event: {}. Skipping", event);
+ }
+ }
+
+ return enabledEvents;
+ }
+
+ private static TraceLevel getTracingLevel(ServerCnx cnx) {
+ // todo: do this less frequently
+ String level =
+ cnx.getBrokerService()
+ .getPulsar()
+ .getConfiguration()
+ .getProperties()
+ .getProperty("tracing_level", defaultTraceLevel.toString());
+ try {
+ return TraceLevel.valueOf(level);
+ } catch (IllegalArgumentException e) {
+ log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+ return defaultTraceLevel;
+ }
+ }
+
+ private static void populateConnectionDetails(
+ TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+ if (cnx == null) {
+ traceDetails.put("ServerCnx", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
+ break;
+ case BASIC:
+ populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+
+ traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
+ traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+ break;
+ case FULL:
+ populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+ populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
+
+ traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
+ traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod());
+ traceDetails.put(
+ "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populateSubscriptionDetails(
+ TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+ if (sub == null) {
+ traceDetails.put("Subscription", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("Subscription.name", sub.getName());
+ traceDetails.put(
+ "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("Subscription.type", sub.getType().name());
+ break;
+ case BASIC:
+ populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+
+ if (sub.getConsumers() != null) {
+ traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
+ traceDetails.put(
+ "Subscription.namesOfConsumers",
+ sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
+ }
+
+ break;
+ case FULL:
+ populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+ populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
+
+ traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populateConsumerDetails(
+ TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+ if (consumer == null) {
+ traceDetails.put("Consumer", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("Consumer.name", consumer.consumerName());
+ traceDetails.put("Consumer.consumerId", consumer.consumerId());
+ if (consumer.getSubscription() != null) {
+ traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName());
+ traceDetails.put(
+ "Consumer.topicName",
+ TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+ }
+ break;
+ case BASIC:
+ populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+
+ traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
+ traceDetails.put("Consumer.subType", consumer.subType().name());
+ traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
+ break;
+ case FULL:
+ populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+ populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+
+ traceDetails.put("Consumer.metadata", consumer.getMetadata());
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populateProducerDetails(
+ TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+ if (producer == null) {
+ traceDetails.put("Producer", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("Producer.producerId", producer.getProducerId());
+ traceDetails.put("Producer.producerName", producer.getProducerName());
+ traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
+ if (producer.getTopic() != null) {
+ traceDetails.put(
+ "Producer.topicName",
+ TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+ }
+ break;
+ case BASIC:
+ populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+
+ traceDetails.put("Producer.clientAddress", producer.getClientAddress());
+ break;
+ case FULL:
+ populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+ populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+
+ traceDetails.put("Producer.metadata", producer.getMetadata());
+ if (producer.getSchemaVersion() != null) {
+ traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString());
+ }
+ traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populateMessageMetadataDetails(
+ TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+ if (msgMetadata == null) {
+ traceDetails.put("MessageMetadata", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
+ traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
+ traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
+ break;
+ case BASIC:
+ populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+
+ traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
+ traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
+ traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+ break;
+ case FULL:
+ populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+ populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
+
+ traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
+ traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime());
+ traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom());
+ traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populateEntryDetails(
+ TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+ if (entry == null) {
+ traceDetails.put("Entry", "null");
+ return;
+ }
+
+ switch (level) {
+ case MIMIMAL:
+ traceDetails.put("Entry.ledgerId", entry.getLedgerId());
+ traceDetails.put("Entry.entryId", entry.getEntryId());
+ break;
+ case BASIC:
+ populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+
+ traceDetails.put("Entry.length", entry.getLength());
+ break;
+ case FULL:
+ populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+ populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+
+ traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
+ break;
+ default:
+ log.debug("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ private static void populatePublishContext(
+ Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+ traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage());
+ traceDetails.put("publishContext.isChunked", publishContext.isChunked());
+ traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages());
+
+ traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp());
+ traceDetails.put("publishContext.msgSize", publishContext.getMsgSize());
+ traceDetails.put("publishContext.producerName", publishContext.getProducerName());
+ traceDetails.put(
+ "publishContext.originalProducerName", publishContext.getOriginalProducerName());
+ traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId());
+ traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId());
+ }
+
+ private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
+ if (buf == null) return;
+
+ if (buf.readableBytes() < MAX_DATA_LENGTH) {
+ traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+ } else {
+ traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+ }
+ }
+
+ private static TraceLevel getTracingLevel(Subscription sub) {
+ if (sub == null
+ || sub.getSubscriptionProperties() == null
+ || !sub.getSubscriptionProperties().containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
+ try {
+ return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+ } catch (IllegalArgumentException e) {
+ log.debug(
+ "Invalid tracing level: {}. Setting to NONE for subscription {}",
+ sub.getSubscriptionProperties().get("trace"),
+ sub);
+ return TraceLevel.NONE;
+ }
+ }
+
+ private static TraceLevel getTracingLevel(Producer producer) {
+ if (producer == null
+ || producer.getMetadata() == null
+ || !producer.getMetadata().containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
+ try {
+ return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+ } catch (IllegalArgumentException e) {
+ log.debug(
+ "Invalid tracing level: {}. Setting to NONE for producer {}",
+ producer.getMetadata().get("trace"),
+ producer);
+ return TraceLevel.NONE;
+ }
+ }
+
+ // private boolean needToTraceTopic(Topic topic) {
+ // if (topic == null) return false;
+ //
+ // // todo: how to read topic props
+ // return true;
+ // }
+
+ /* ***************************
+ ** Administrative events
+ ******************************/
+
+ public void onConnectionCreated(ServerCnx cnx) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ log.info("Connection created: {}", traceDetails);
+ }
+
+ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateProducerDetails(level, producer, traceDetails);
+
+ traceDetails.put("metadata", metadata);
+
+ log.info("Producer created: {}", traceDetails);
+ }
+
+ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateProducerDetails(level, producer, traceDetails);
+
+ traceDetails.put("metadata", metadata);
+
+ log.info("Producer closed: {}", traceDetails);
+ }
+
+ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateConsumerDetails(level, consumer, traceDetails);
+ populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+ traceDetails.put("metadata", metadata);
+
+ log.info("Consumer created: {}", traceDetails);
+ }
+
+ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateConsumerDetails(level, consumer, traceDetails);
+ populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+ traceDetails.put("metadata", metadata);
+
+ log.info("Consumer closed: {}", traceDetails);
+ }
+
+ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ traceDetails.put("command", command.toString());
+ log.info("Pulsar command called: {}", traceDetails);
+ }
+
+ public void onConnectionClosed(ServerCnx cnx) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+ TraceLevel level = getTracingLevel(cnx);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ log.info("Connection closed: {}", traceDetails);
+ }
+
+ /* ***************************
+ ** Message events
+ ******************************/
+
+ public void beforeSendMessage(
+ Subscription subscription,
+ Entry entry,
+ long[] ackSet,
+ MessageMetadata msgMetadata,
+ Consumer consumer) {
+ if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar())
+ .contains(EventReasons.MESSAGE)) return;
+
+ TraceLevel level = getTracingLevel(subscription);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConsumerDetails(level, consumer, traceDetails);
+ populateSubscriptionDetails(level, subscription, traceDetails);
+ populateEntryDetails(level, entry, traceDetails);
+ populateMessageMetadataDetails(level, msgMetadata, traceDetails);
+
+ log.info("Before sending message: {}", traceDetails);
+ }
+
+ public void onMessagePublish(
+ Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
+
+ if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar())
+ .contains(EventReasons.MESSAGE)) return;
+
+ TraceLevel level = getTracingLevel(producer);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateProducerDetails(level, producer, traceDetails);
+
+ traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+
+ populatePublishContext(publishContext, traceDetails);
+
+ log.info("Message publish: {}", traceDetails);
+ }
+
+ public void messageProduced(
+ ServerCnx cnx,
+ Producer producer,
+ long startTimeNs,
+ long ledgerId,
+ long entryId,
+ Topic.PublishContext publishContext) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+ TraceLevel level = getTracingLevel(producer);
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateProducerDetails(level, producer, traceDetails);
+ populatePublishContext(publishContext, traceDetails);
+
+ traceDetails.put("ledgerId", ledgerId);
+ traceDetails.put("entryId", entryId);
+ traceDetails.put("startTimeNs", startTimeNs);
+
+ log.info("Message produced: {}", traceDetails);
+ }
+
+ public void messageDispatched(
+ ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+ TraceLevel level = getTracingLevel(consumer.getSubscription());
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateConsumerDetails(level, consumer, traceDetails);
+ populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+ traceDetails.put("ledgerId", ledgerId);
+ traceDetails.put("entryId", entryId);
+
+ traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+
+ log.info("After dispatching message: {}", traceDetails);
+ }
+
+ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
+ if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+ TraceLevel level = getTracingLevel(consumer.getSubscription());
+ if (level == TraceLevel.NONE) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ populateConnectionDetails(level, cnx, traceDetails);
+ populateConsumerDetails(level, consumer, traceDetails);
+ populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+ traceDetails.put("ack.type", ackCmd.getAckType().name());
+ traceDetails.put("ack.consumerId", ackCmd.getConsumerId());
+ traceDetails.put(
+ "ack.messageIds",
+ ackCmd
+ .getMessageIdsList()
+ .stream()
+ .map(x -> x.getLedgerId() + ":" + x.getEntryId())
+ .collect(Collectors.toList()));
+
+ log.info("Message acked: {}", traceDetails);
+ }
+
+ /* ***************************
+ ** Transaction events
+ ******************************/
+
+ public void txnOpened(long tcId, String txnID) {
+ // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+ Map<String, Object> traceDetails = new TreeMap<>();
+ traceDetails.put("tcId", tcId);
+ traceDetails.put("txnID", txnID);
+
+ log.info("Transaction opened: {}", traceDetails);
+ // }
+ }
+
+ public void txnEnded(String txnID, long txnAction) {
+ // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+ Map<String, Object> traceDetails = new TreeMap<>();
+ traceDetails.put("txnID", txnID);
+ traceDetails.put("txnAction", txnAction);
+
+ log.info("Transaction closed: {}", traceDetails);
+ // }
+ }
+
+ /* ***************************
+ ** Servlet events
+ ******************************/
+
+ public void onWebserviceRequest(ServletRequest request)
+ throws IOException, ServletException, InterceptException {
+ // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
+ // log.info("onWebserviceRequest: Tracing servlet requests not supported");
+ // }
+ }
+
+ public void onWebserviceResponse(ServletRequest request, ServletResponse response)
+ throws IOException, ServletException {
+ // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
+ // log.info("onWebserviceResponse: Tracing servlet requests not supported");
+ // }
+ }
+
+ // not needed
+ // public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+}
diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
new file mode 100644
index 00000000..38748bb9
--- /dev/null
+++ b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
@@ -0,0 +1,3 @@
+interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing
+name: pulsar-tracing
+description: Starlight for JMS - support for server side tracing
\ No newline at end of file
diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-tracing/src/test/resources/log4j2.xml
new file mode 100644
index 00000000..485f6705
--- /dev/null
+++ b/pulsar-tracing/src/test/resources/log4j2.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Copyright DataStax, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="DEBUG">
+ <AppenderRef ref="Console"/>
+ </Root>
+ <logger name="org.apache.bookkeeper" level="error" additivity="true"/>
+ <logger name="org.apache.bookkeeper.mledger" level="info" additivity="true"/>
+ <logger name="org.apache.pulsar" level="info" additivity="true"/>
+ <logger name="org.apache.zookeeper" level="error" additivity="true"/>
+ <logger name="org.apache.curator" level="error" additivity="true"/>
+ <logger name="org.eclipse.jetty" level="error" additivity="true"/>
+ <logger name="org.glassfish.jersey" level="error" additivity="false"/>
+ <logger name="com.datastax.oss.pulsar.jms" level="debug" additivity="true"/>
+ <logger name="com.datastax.oss.pulsar.jms.utils" level="info" additivity="false"/>
+ <logger name="io.netty" level="error" additivity="true"/>
+ <logger name="org.asynchttpclient" level="error" additivity="true"/>
+ </Loggers>
+</Configuration>
From 91e51864fd52f409366310923ada743e71833bdd Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 11:12:41 -0700
Subject: [PATCH 02/29] various renamings and tweaks
---
pom.xml | 2 +-
.../pom.xml | 4 +-
.../pulsar/jms}/tracing/BrokerTracing.java | 91 ++++++++++---------
.../META-INF/services/broker_interceptor.yml | 3 +
.../src/test/resources/log4j2.xml | 0
.../META-INF/services/broker_interceptor.yml | 3 -
6 files changed, 53 insertions(+), 50 deletions(-)
rename {pulsar-tracing => pulsar-jms-tracing}/pom.xml (95%)
rename {pulsar-tracing/src/main/java/com/datastax/oss/pulsar => pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms}/tracing/BrokerTracing.java (89%)
create mode 100644 pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
rename {pulsar-tracing => pulsar-jms-tracing}/src/test/resources/log4j2.xml (100%)
delete mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
diff --git a/pom.xml b/pom.xml
index 8f9c2802..a24e298a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@
<modules>
<module>activemq-filters</module>
<module>pulsar-jms-filters</module>
- <module>pulsar-tracing</module>
+ <module>pulsar-jms-tracing</module>
<module>pulsar-jms-admin-api</module>
<module>pulsar-jms</module>
<module>resource-adapter</module>
diff --git a/pulsar-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
similarity index 95%
rename from pulsar-tracing/pom.xml
rename to pulsar-jms-tracing/pom.xml
index e4a65161..2d4536d8 100644
--- a/pulsar-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -23,7 +23,7 @@
<version>4.1.1-alpha-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>pulsar-tracing</artifactId>
+ <artifactId>pulsar-jms-tracing</artifactId>
<packaging>jar</packaging>
<name>DataStax Starlight for JMS - Broker Side Filters</name>
<properties>
@@ -64,7 +64,7 @@
<version>1.3.2</version>
<extensions>true</extensions>
<configuration>
- <finalName>pulsar-tracing-${project.version}</finalName>
+ <finalName>pulsar-jms-tracing-${project.version}</finalName>
<classifier>nar</classifier>
</configuration>
<executions>
diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
similarity index 89%
rename from pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
rename to pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index e18e2cec..82e6ec3b 100644
--- a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.datastax.oss.pulsar.tracing;
+package com.datastax.oss.pulsar.jms.tracing;
import io.netty.buffer.ByteBuf;
import java.io.IOException;
@@ -44,6 +44,8 @@
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
+ public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
static final int MAX_DATA_LENGTH = 1024;
public enum EventReasons {
@@ -55,7 +57,7 @@ public enum EventReasons {
public enum TraceLevel {
NONE,
- MIMIMAL,
+ MINIMAL,
BASIC,
FULL
}
@@ -79,7 +81,8 @@ private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
// todo: do this less frequently
- String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", "");
+ String events =
+ pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
Set<EventReasons> enabledEvents = new HashSet<>();
@@ -101,7 +104,7 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
.getPulsar()
.getConfiguration()
.getProperties()
- .getProperty("tracing_level", defaultTraceLevel.toString());
+ .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
try {
return TraceLevel.valueOf(level);
} catch (IllegalArgumentException e) {
@@ -118,17 +121,17 @@ private static void populateConnectionDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
break;
case BASIC:
- populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+ populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
break;
case FULL:
- populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+ populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
@@ -137,7 +140,7 @@ private static void populateConnectionDetails(
"ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -150,14 +153,14 @@ private static void populateSubscriptionDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("Subscription.name", sub.getName());
traceDetails.put(
"Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
traceDetails.put("Subscription.type", sub.getType().name());
break;
case BASIC:
- populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+ populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
if (sub.getConsumers() != null) {
traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
@@ -168,13 +171,13 @@ private static void populateSubscriptionDetails(
break;
case FULL:
- populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+ populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -187,7 +190,7 @@ private static void populateConsumerDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("Consumer.name", consumer.consumerName());
traceDetails.put("Consumer.consumerId", consumer.consumerId());
if (consumer.getSubscription() != null) {
@@ -198,20 +201,20 @@ private static void populateConsumerDetails(
}
break;
case BASIC:
- populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+ populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
traceDetails.put("Consumer.subType", consumer.subType().name());
traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
break;
case FULL:
- populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+ populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
traceDetails.put("Consumer.metadata", consumer.getMetadata());
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -224,7 +227,7 @@ private static void populateProducerDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("Producer.producerId", producer.getProducerId());
traceDetails.put("Producer.producerName", producer.getProducerName());
traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
@@ -235,12 +238,12 @@ private static void populateProducerDetails(
}
break;
case BASIC:
- populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+ populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
traceDetails.put("Producer.clientAddress", producer.getClientAddress());
break;
case FULL:
- populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+ populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
traceDetails.put("Producer.metadata", producer.getMetadata());
@@ -250,7 +253,7 @@ private static void populateProducerDetails(
traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -263,20 +266,20 @@ private static void populateMessageMetadataDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
break;
case BASIC:
- populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+ populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
break;
case FULL:
- populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+ populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
@@ -285,7 +288,7 @@ private static void populateMessageMetadataDetails(
traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -298,23 +301,23 @@ private static void populateEntryDetails(
}
switch (level) {
- case MIMIMAL:
+ case MINIMAL:
traceDetails.put("Entry.ledgerId", entry.getLedgerId());
traceDetails.put("Entry.entryId", entry.getEntryId());
break;
case BASIC:
- populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+ populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
traceDetails.put("Entry.length", entry.getLength());
break;
case FULL:
- populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+ populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
break;
default:
- log.debug("Unknown tracing level: {}", level);
+ log.warn("Unknown tracing level: {}", level);
break;
}
}
@@ -353,7 +356,7 @@ private static TraceLevel getTracingLevel(Subscription sub) {
try {
return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
} catch (IllegalArgumentException e) {
- log.debug(
+ log.warn(
"Invalid tracing level: {}. Setting to NONE for subscription {}",
sub.getSubscriptionProperties().get("trace"),
sub);
@@ -370,7 +373,7 @@ private static TraceLevel getTracingLevel(Producer producer) {
try {
return TraceLevel.valueOf(producer.getMetadata().get("trace"));
} catch (IllegalArgumentException e) {
- log.debug(
+ log.warn(
"Invalid tracing level: {}. Setting to NONE for producer {}",
producer.getMetadata().get("trace"),
producer);
@@ -397,7 +400,7 @@ public void onConnectionCreated(ServerCnx cnx) {
Map<String, Object> traceDetails = new TreeMap<>();
populateConnectionDetails(level, cnx, traceDetails);
- log.info("Connection created: {}", traceDetails);
+ tracer.info("Connection created: {}", traceDetails);
}
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -412,7 +415,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
traceDetails.put("metadata", metadata);
- log.info("Producer created: {}", traceDetails);
+ tracer.info("Producer created: {}", traceDetails);
}
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -427,7 +430,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
traceDetails.put("metadata", metadata);
- log.info("Producer closed: {}", traceDetails);
+ tracer.info("Producer closed: {}", traceDetails);
}
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -443,7 +446,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
traceDetails.put("metadata", metadata);
- log.info("Consumer created: {}", traceDetails);
+ tracer.info("Consumer created: {}", traceDetails);
}
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -459,7 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
traceDetails.put("metadata", metadata);
- log.info("Consumer closed: {}", traceDetails);
+ tracer.info("Consumer closed: {}", traceDetails);
}
public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
@@ -471,7 +474,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
Map<String, Object> traceDetails = new TreeMap<>();
populateConnectionDetails(level, cnx, traceDetails);
traceDetails.put("command", command.toString());
- log.info("Pulsar command called: {}", traceDetails);
+ tracer.info("Pulsar command called: {}", traceDetails);
}
public void onConnectionClosed(ServerCnx cnx) {
@@ -482,7 +485,7 @@ public void onConnectionClosed(ServerCnx cnx) {
Map<String, Object> traceDetails = new TreeMap<>();
populateConnectionDetails(level, cnx, traceDetails);
- log.info("Connection closed: {}", traceDetails);
+ tracer.info("Connection closed: {}", traceDetails);
}
/* ***************************
@@ -507,7 +510,7 @@ public void beforeSendMessage(
populateEntryDetails(level, entry, traceDetails);
populateMessageMetadataDetails(level, msgMetadata, traceDetails);
- log.info("Before sending message: {}", traceDetails);
+ tracer.info("Before sending message: {}", traceDetails);
}
public void onMessagePublish(
@@ -526,7 +529,7 @@ public void onMessagePublish(
populatePublishContext(publishContext, traceDetails);
- log.info("Message publish: {}", traceDetails);
+ tracer.info("Message publish: {}", traceDetails);
}
public void messageProduced(
@@ -551,7 +554,7 @@ public void messageProduced(
traceDetails.put("entryId", entryId);
traceDetails.put("startTimeNs", startTimeNs);
- log.info("Message produced: {}", traceDetails);
+ tracer.info("Message produced: {}", traceDetails);
}
public void messageDispatched(
@@ -571,7 +574,7 @@ public void messageDispatched(
traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
- log.info("After dispatching message: {}", traceDetails);
+ tracer.info("After dispatching message: {}", traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -595,7 +598,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
.map(x -> x.getLedgerId() + ":" + x.getEntryId())
.collect(Collectors.toList()));
- log.info("Message acked: {}", traceDetails);
+ tracer.info("Message acked: {}", traceDetails);
}
/* ***************************
@@ -608,7 +611,7 @@ public void txnOpened(long tcId, String txnID) {
traceDetails.put("tcId", tcId);
traceDetails.put("txnID", txnID);
- log.info("Transaction opened: {}", traceDetails);
+ tracer.info("Transaction opened: {}", traceDetails);
// }
}
@@ -618,7 +621,7 @@ public void txnEnded(String txnID, long txnAction) {
traceDetails.put("txnID", txnID);
traceDetails.put("txnAction", txnAction);
- log.info("Transaction closed: {}", traceDetails);
+ tracer.info("Transaction closed: {}", traceDetails);
// }
}
diff --git a/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
new file mode 100644
index 00000000..a47fa05f
--- /dev/null
+++ b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
@@ -0,0 +1,3 @@
+interceptorClass: com.datastax.oss.pulsar.jms.tracing.BrokerTracing
+name: jms-tracing
+description: Starlight for JMS - support for server side tracing
\ No newline at end of file
diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-jms-tracing/src/test/resources/log4j2.xml
similarity index 100%
rename from pulsar-tracing/src/test/resources/log4j2.xml
rename to pulsar-jms-tracing/src/test/resources/log4j2.xml
diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
deleted file mode 100644
index 38748bb9..00000000
--- a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing
-name: pulsar-tracing
-description: Starlight for JMS - support for server side tracing
\ No newline at end of file
From ea914204b8653be053ae27499e33f6e022627211 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 12:16:28 -0700
Subject: [PATCH 03/29] traces as json
---
pulsar-jms-tracing/pom.xml | 4 +
.../oss/pulsar/jms/tracing/BrokerTracing.java | 315 +++++++++++-------
2 files changed, 200 insertions(+), 119 deletions(-)
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index 2d4536d8..c60c0b4a 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -30,6 +30,10 @@
<build.dir>${project.build.directory}</build.dir>
</properties>
<dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 82e6ec3b..dd41a4c4 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,6 +15,9 @@
*/
package com.datastax.oss.pulsar.jms.tracing;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
import io.netty.buffer.ByteBuf;
import java.io.IOException;
import java.util.HashSet;
@@ -44,7 +47,12 @@
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
- public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+ private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+ private static final ObjectMapper mapper =
+ new ObjectMapper()
+ .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
+ .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
static final int MAX_DATA_LENGTH = 1024;
@@ -65,6 +73,19 @@ public enum TraceLevel {
// private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+ public static void trace(String message, Map<String, Object> traceDetails) {
+ Map<String, Object> trace = new TreeMap<>();
+ trace.put("message", message);
+ trace.put("details", traceDetails);
+
+ try {
+ String loggableJsonString = mapper.writeValueAsString(trace);
+ tracer.info(loggableJsonString);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public void initialize(PulsarService pulsarService) {
// defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
// defaultEnabledEvents.add(EventReasons.MESSAGE);
@@ -113,31 +134,39 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
}
}
+ public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+ if (cnx == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateConnectionDetails(level, cnx, details);
+ return details;
+ }
+
private static void populateConnectionDetails(
TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
if (cnx == null) {
- traceDetails.put("ServerCnx", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
+ traceDetails.put("clientAddress", cnx.clientAddress().toString());
break;
case BASIC:
populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
- traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
- traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+ traceDetails.put("clientVersion", cnx.getClientVersion());
+ traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
break;
case FULL:
populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
- traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
- traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod());
- traceDetails.put(
- "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+ traceDetails.put("authRole", cnx.getAuthRole());
+ traceDetails.put("authMethod", cnx.getAuthMethod());
+ traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -145,27 +174,35 @@ private static void populateConnectionDetails(
}
}
+ public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+ if (sub == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateSubscriptionDetails(level, sub, details);
+ return details;
+ }
+
private static void populateSubscriptionDetails(
TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
if (sub == null) {
- traceDetails.put("Subscription", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("Subscription.name", sub.getName());
- traceDetails.put(
- "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- traceDetails.put("Subscription.type", sub.getType().name());
+ traceDetails.put("name", sub.getName());
+ traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("type", sub.getType().name());
break;
case BASIC:
populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
if (sub.getConsumers() != null) {
- traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
+ traceDetails.put("numberOfConsumers", sub.getConsumers().size());
traceDetails.put(
- "Subscription.namesOfConsumers",
+ "namesOfConsumers",
sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
}
@@ -174,7 +211,7 @@ private static void populateSubscriptionDetails(
populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
- traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
+ traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -182,36 +219,45 @@ private static void populateSubscriptionDetails(
}
}
+ public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+ if (consumer == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateConsumerDetails(level, consumer, details);
+ return details;
+ }
+
private static void populateConsumerDetails(
TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
if (consumer == null) {
- traceDetails.put("Consumer", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("Consumer.name", consumer.consumerName());
- traceDetails.put("Consumer.consumerId", consumer.consumerId());
+ traceDetails.put("name", consumer.consumerName());
+ traceDetails.put("consumerId", consumer.consumerId());
if (consumer.getSubscription() != null) {
- traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName());
+ traceDetails.put("subscriptionName", consumer.getSubscription().getName());
traceDetails.put(
- "Consumer.topicName",
+ "topicName",
TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
}
break;
case BASIC:
populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
- traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
- traceDetails.put("Consumer.subType", consumer.subType().name());
- traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
+ traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+ traceDetails.put("subType", consumer.subType().name());
+ traceDetails.put("clientAddress", consumer.getClientAddress());
break;
case FULL:
populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
- traceDetails.put("Consumer.metadata", consumer.getMetadata());
+ traceDetails.put("metadata", consumer.getMetadata());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -219,38 +265,46 @@ private static void populateConsumerDetails(
}
}
+ public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+ if (producer == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateProducerDetails(level, producer, details);
+ return details;
+ }
+
private static void populateProducerDetails(
TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
if (producer == null) {
- traceDetails.put("Producer", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("Producer.producerId", producer.getProducerId());
- traceDetails.put("Producer.producerName", producer.getProducerName());
- traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
+ traceDetails.put("producerId", producer.getProducerId());
+ traceDetails.put("producerName", producer.getProducerName());
+ traceDetails.put("accessMode", producer.getAccessMode().name());
if (producer.getTopic() != null) {
traceDetails.put(
- "Producer.topicName",
- TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+ "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
}
break;
case BASIC:
populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
- traceDetails.put("Producer.clientAddress", producer.getClientAddress());
+ traceDetails.put("clientAddress", producer.getClientAddress());
break;
case FULL:
populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
- traceDetails.put("Producer.metadata", producer.getMetadata());
+ traceDetails.put("metadata", producer.getMetadata());
if (producer.getSchemaVersion() != null) {
- traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString());
+ traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
}
- traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
+ traceDetails.put("remoteCluster", producer.getRemoteCluster());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -258,34 +312,44 @@ private static void populateProducerDetails(
}
}
+ public static Map<String, Object> getMessageMetadataDetails(
+ TraceLevel level, MessageMetadata msgMetadata) {
+ if (msgMetadata == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateMessageMetadataDetails(level, msgMetadata, details);
+ return details;
+ }
+
private static void populateMessageMetadataDetails(
TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
if (msgMetadata == null) {
- traceDetails.put("MessageMetadata", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
- traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
- traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
+ traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+ traceDetails.put("producerName", msgMetadata.getProducerName());
+ traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
break;
case BASIC:
populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
- traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
- traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
- traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+ traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+ traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+ traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
break;
case FULL:
populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
- traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
- traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime());
- traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom());
- traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
+ traceDetails.put("publishTime", msgMetadata.getPublishTime());
+ traceDetails.put("eventTime", msgMetadata.getEventTime());
+ traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+ traceDetails.put("uuid", msgMetadata.getUuid());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -293,28 +357,37 @@ private static void populateMessageMetadataDetails(
}
}
+ public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+ if (entry == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateEntryDetails(level, entry, details);
+ return details;
+ }
+
private static void populateEntryDetails(
TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
if (entry == null) {
- traceDetails.put("Entry", "null");
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("Entry.ledgerId", entry.getLedgerId());
- traceDetails.put("Entry.entryId", entry.getEntryId());
+ traceDetails.put("ledgerId", entry.getLedgerId());
+ traceDetails.put("entryId", entry.getEntryId());
break;
case BASIC:
populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
- traceDetails.put("Entry.length", entry.getLength());
+ traceDetails.put("length", entry.getLength());
break;
case FULL:
populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
- traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
+ traceByteBuf("data", entry.getDataBuffer(), traceDetails);
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -322,19 +395,28 @@ private static void populateEntryDetails(
}
}
+ public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+ if (publishContext == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populatePublishContext(publishContext, details);
+ return details;
+ }
+
private static void populatePublishContext(
Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
- traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage());
- traceDetails.put("publishContext.isChunked", publishContext.isChunked());
- traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages());
+ traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+ traceDetails.put("isChunked", publishContext.isChunked());
+ traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
- traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp());
- traceDetails.put("publishContext.msgSize", publishContext.getMsgSize());
- traceDetails.put("publishContext.producerName", publishContext.getProducerName());
- traceDetails.put(
- "publishContext.originalProducerName", publishContext.getOriginalProducerName());
- traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId());
- traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId());
+ traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
+ traceDetails.put("msgSize", publishContext.getMsgSize());
+ traceDetails.put("producerName", publishContext.getProducerName());
+ traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+ traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+ traceDetails.put("sequenceId", publishContext.getSequenceId());
}
private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
@@ -399,8 +481,8 @@ public void onConnectionCreated(ServerCnx cnx) {
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- tracer.info("Connection created: {}", traceDetails);
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ trace("Connection created", traceDetails);
}
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -410,12 +492,11 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateProducerDetails(level, producer, traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("producer", getProducerDetails(level, producer));
traceDetails.put("metadata", metadata);
- tracer.info("Producer created: {}", traceDetails);
+ trace("Producer created", traceDetails);
}
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -425,12 +506,11 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateProducerDetails(level, producer, traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("producer", getProducerDetails(level, producer));
traceDetails.put("metadata", metadata);
- tracer.info("Producer closed: {}", traceDetails);
+ trace("Producer closed", traceDetails);
}
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -440,13 +520,12 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateConsumerDetails(level, consumer, traceDetails);
- populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("consumer", getConsumerDetails(level, consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
traceDetails.put("metadata", metadata);
- tracer.info("Consumer created: {}", traceDetails);
+ trace("Consumer created", traceDetails);
}
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -456,13 +535,12 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateConsumerDetails(level, consumer, traceDetails);
- populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("consumer", getConsumerDetails(level, consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
traceDetails.put("metadata", metadata);
- tracer.info("Consumer closed: {}", traceDetails);
+ trace("Consumer closed", traceDetails);
}
public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
@@ -472,9 +550,10 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
traceDetails.put("command", command.toString());
- tracer.info("Pulsar command called: {}", traceDetails);
+
+ trace("Pulsar command called", traceDetails);
}
public void onConnectionClosed(ServerCnx cnx) {
@@ -484,8 +563,9 @@ public void onConnectionClosed(ServerCnx cnx) {
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- tracer.info("Connection closed: {}", traceDetails);
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+
+ trace("Connection closed", traceDetails);
}
/* ***************************
@@ -505,12 +585,12 @@ public void beforeSendMessage(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConsumerDetails(level, consumer, traceDetails);
- populateSubscriptionDetails(level, subscription, traceDetails);
- populateEntryDetails(level, entry, traceDetails);
- populateMessageMetadataDetails(level, msgMetadata, traceDetails);
+ traceDetails.put("subscription", getSubscriptionDetails(level, subscription));
+ traceDetails.put("consumer", getConsumerDetails(level, consumer));
+ traceDetails.put("entry", getEntryDetails(level, entry));
+ traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
- tracer.info("Before sending message: {}", traceDetails);
+ trace("Before sending message", traceDetails);
}
public void onMessagePublish(
@@ -523,13 +603,11 @@ public void onMessagePublish(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateProducerDetails(level, producer, traceDetails);
-
+ traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
- populatePublishContext(publishContext, traceDetails);
-
- tracer.info("Message publish: {}", traceDetails);
+ trace("Message publish", traceDetails);
}
public void messageProduced(
@@ -545,16 +623,14 @@ public void messageProduced(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
-
- populateConnectionDetails(level, cnx, traceDetails);
- populateProducerDetails(level, producer, traceDetails);
- populatePublishContext(publishContext, traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("ledgerId", ledgerId);
traceDetails.put("entryId", entryId);
traceDetails.put("startTimeNs", startTimeNs);
- tracer.info("Message produced: {}", traceDetails);
+ trace("Message produced", traceDetails);
}
public void messageDispatched(
@@ -565,16 +641,14 @@ public void messageDispatched(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateConsumerDetails(level, consumer, traceDetails);
- populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("consumer", getConsumerDetails(level, consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
traceDetails.put("ledgerId", ledgerId);
traceDetails.put("entryId", entryId);
-
traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
- tracer.info("After dispatching message: {}", traceDetails);
+ trace("After dispatching message", traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -584,21 +658,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- populateConnectionDetails(level, cnx, traceDetails);
- populateConsumerDetails(level, consumer, traceDetails);
- populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
- traceDetails.put("ack.type", ackCmd.getAckType().name());
- traceDetails.put("ack.consumerId", ackCmd.getConsumerId());
- traceDetails.put(
- "ack.messageIds",
+ traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("consumer", getConsumerDetails(level, consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+
+ Map<String, Object> ackDetails = new TreeMap<>();
+ ackDetails.put("type", ackCmd.getAckType().name());
+ ackDetails.put("consumerId", ackCmd.getConsumerId());
+ ackDetails.put(
+ "messageIds",
ackCmd
.getMessageIdsList()
.stream()
.map(x -> x.getLedgerId() + ":" + x.getEntryId())
.collect(Collectors.toList()));
- tracer.info("Message acked: {}", traceDetails);
+ traceDetails.put("ack", ackDetails);
+
+ trace("Message acked", traceDetails);
}
/* ***************************
@@ -611,7 +688,7 @@ public void txnOpened(long tcId, String txnID) {
traceDetails.put("tcId", tcId);
traceDetails.put("txnID", txnID);
- tracer.info("Transaction opened: {}", traceDetails);
+ trace("Transaction opened", traceDetails);
// }
}
@@ -621,7 +698,7 @@ public void txnEnded(String txnID, long txnAction) {
traceDetails.put("txnID", txnID);
traceDetails.put("txnAction", txnAction);
- tracer.info("Transaction closed: {}", traceDetails);
+ trace("Transaction closed", traceDetails);
// }
}
From 40f47848b7de41987ffc56df93e14dd149c0a886 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 15:58:49 -0700
Subject: [PATCH 04/29] refactoring, cache some of configuration
---
pom.xml | 6 +
pulsar-jms-tracing/pom.xml | 4 +
.../oss/pulsar/jms/tracing/BrokerTracing.java | 436 ++++--------------
.../oss/pulsar/jms/tracing/TracingUtils.java | 361 +++++++++++++++
4 files changed, 449 insertions(+), 358 deletions(-)
create mode 100644 pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
diff --git a/pom.xml b/pom.xml
index a24e298a..0b10c2b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,6 +65,7 @@
<surefire.version>3.1.0</surefire.version>
<jackson.version>2.14.2</jackson.version>
<gson.version>2.8.9</gson.version>
+ <guava.version>32.1.2-jre</guava.version>
<commons-compress.version>1.21</commons-compress.version>
<awaitility.version>4.0.3</awaitility.version>
<maven.antrun.plugin.version>3.1.0</maven.antrun.plugin.version>
@@ -107,6 +108,11 @@
<artifactId>slf4j-simple</artifactId>
<version>${slf4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index c60c0b4a..da7d2dda 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -34,6 +34,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dd41a4c4..d3b7f248 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,22 +15,33 @@
*/
package com.datastax.oss.pulsar.jms.tracing;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import io.netty.buffer.ByteBuf;
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Entry;
-import org.apache.commons.codec.binary.Hex;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.intercept.BrokerInterceptor;
import org.apache.pulsar.broker.service.Consumer;
@@ -42,19 +53,60 @@
import org.apache.pulsar.common.api.proto.CommandAck;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.intercept.InterceptException;
-import org.apache.pulsar.common.naming.TopicName;
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
- private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
-
- private static final ObjectMapper mapper =
- new ObjectMapper()
- .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
- .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
-
- static final int MAX_DATA_LENGTH = 1024;
+ private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
+ CacheBuilder.newBuilder()
+ .maximumSize(10)
+ .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+ .build(
+ new CacheLoader<PulsarService, Set<EventReasons>>() {
+ public Set<EventReasons> load(PulsarService pulsar) {
+ // dynamic config can get reloaded without service restart
+ String events =
+ pulsar
+ .getConfiguration()
+ .getProperties()
+ .getProperty("jmsTracingEventList", "");
+ log.debug("read jmsTracingEventList: {}", events);
+
+ Set<EventReasons> enabledEvents = new HashSet<>();
+
+ for (String event : events.split(",")) {
+ try {
+ enabledEvents.add(EventReasons.valueOf(event.trim()));
+ } catch (IllegalArgumentException e) {
+ log.warn("Invalid event: {}. Skipping", event);
+ }
+ }
+
+ return enabledEvents;
+ }
+ });
+
+ private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
+ CacheBuilder.newBuilder()
+ .maximumSize(10)
+ .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+ .build(
+ new CacheLoader<PulsarService, TraceLevel>() {
+ public TraceLevel load(PulsarService pulsar) {
+ String level =
+ pulsar
+ .getConfiguration()
+ .getProperties()
+ .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+ try {
+ return TraceLevel.valueOf(level);
+ } catch (IllegalArgumentException e) {
+ log.warn(
+ "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+ return defaultTraceLevel;
+ }
+ }
+ });
public enum EventReasons {
ADMINISTRATIVE,
@@ -63,372 +115,40 @@ public enum EventReasons {
SERVLET,
}
- public enum TraceLevel {
- NONE,
- MINIMAL,
- BASIC,
- FULL
- }
-
- // private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
- public static void trace(String message, Map<String, Object> traceDetails) {
- Map<String, Object> trace = new TreeMap<>();
- trace.put("message", message);
- trace.put("details", traceDetails);
-
- try {
- String loggableJsonString = mapper.writeValueAsString(trace);
- tracer.info(loggableJsonString);
- } catch (JsonProcessingException e) {
- throw new RuntimeException(e);
- }
- }
-
public void initialize(PulsarService pulsarService) {
- // defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
- // defaultEnabledEvents.add(EventReasons.MESSAGE);
- // defaultEnabledEvents.add(EventReasons.TRANSACTION);
+ log.info("Initializing BrokerTracing");
}
@Override
- public void close() {}
+ public void close() {
+ log.info("Closing BrokerTracing");
+ }
private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
return getEnabledEvents(cnx.getBrokerService().getPulsar());
}
private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
- // todo: do this less frequently
-
- String events =
- pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
-
- Set<EventReasons> enabledEvents = new HashSet<>();
-
- for (String event : events.split(",")) {
- try {
- enabledEvents.add(EventReasons.valueOf(event.trim()));
- } catch (IllegalArgumentException e) {
- log.warn("Invalid event: {}. Skipping", event);
- }
+ try {
+ return jmsTracingEventList.get(pulsar);
+ } catch (ExecutionException e) {
+ log.error("Error getting enabled events", e);
+ return new HashSet<>();
}
-
- return enabledEvents;
}
private static TraceLevel getTracingLevel(ServerCnx cnx) {
- // todo: do this less frequently
- String level =
- cnx.getBrokerService()
- .getPulsar()
- .getConfiguration()
- .getProperties()
- .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+
try {
- return TraceLevel.valueOf(level);
- } catch (IllegalArgumentException e) {
- log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+ return traceLevelForService.get(cnx.getBrokerService().getPulsar());
+ } catch (ExecutionException e) {
+ log.error("Error getting tracing level", e);
return defaultTraceLevel;
}
}
- public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
- if (cnx == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateConnectionDetails(level, cnx, details);
- return details;
- }
-
- private static void populateConnectionDetails(
- TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
- if (cnx == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("clientAddress", cnx.clientAddress().toString());
- break;
- case BASIC:
- populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
-
- traceDetails.put("clientVersion", cnx.getClientVersion());
- traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
- break;
- case FULL:
- populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
- populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
-
- traceDetails.put("authRole", cnx.getAuthRole());
- traceDetails.put("authMethod", cnx.getAuthMethod());
- traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
- if (sub == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateSubscriptionDetails(level, sub, details);
- return details;
- }
-
- private static void populateSubscriptionDetails(
- TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
- if (sub == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("name", sub.getName());
- traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- traceDetails.put("type", sub.getType().name());
- break;
- case BASIC:
- populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
-
- if (sub.getConsumers() != null) {
- traceDetails.put("numberOfConsumers", sub.getConsumers().size());
- traceDetails.put(
- "namesOfConsumers",
- sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
- }
-
- break;
- case FULL:
- populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
- populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
-
- traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
- if (consumer == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateConsumerDetails(level, consumer, details);
- return details;
- }
-
- private static void populateConsumerDetails(
- TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
- if (consumer == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("name", consumer.consumerName());
- traceDetails.put("consumerId", consumer.consumerId());
- if (consumer.getSubscription() != null) {
- traceDetails.put("subscriptionName", consumer.getSubscription().getName());
- traceDetails.put(
- "topicName",
- TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
- }
- break;
- case BASIC:
- populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
-
- traceDetails.put("priorityLevel", consumer.getPriorityLevel());
- traceDetails.put("subType", consumer.subType().name());
- traceDetails.put("clientAddress", consumer.getClientAddress());
- break;
- case FULL:
- populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
- populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
-
- traceDetails.put("metadata", consumer.getMetadata());
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
- if (producer == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateProducerDetails(level, producer, details);
- return details;
- }
-
- private static void populateProducerDetails(
- TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
- if (producer == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("producerId", producer.getProducerId());
- traceDetails.put("producerName", producer.getProducerName());
- traceDetails.put("accessMode", producer.getAccessMode().name());
- if (producer.getTopic() != null) {
- traceDetails.put(
- "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
- }
- break;
- case BASIC:
- populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
-
- traceDetails.put("clientAddress", producer.getClientAddress());
- break;
- case FULL:
- populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
- populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
-
- traceDetails.put("metadata", producer.getMetadata());
- if (producer.getSchemaVersion() != null) {
- traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
- }
- traceDetails.put("remoteCluster", producer.getRemoteCluster());
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getMessageMetadataDetails(
- TraceLevel level, MessageMetadata msgMetadata) {
- if (msgMetadata == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateMessageMetadataDetails(level, msgMetadata, details);
- return details;
- }
-
- private static void populateMessageMetadataDetails(
- TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
- if (msgMetadata == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("sequenceId", msgMetadata.getSequenceId());
- traceDetails.put("producerName", msgMetadata.getProducerName());
- traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
- break;
- case BASIC:
- populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
-
- traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
- traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
- traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
- break;
- case FULL:
- populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
- populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
-
- traceDetails.put("publishTime", msgMetadata.getPublishTime());
- traceDetails.put("eventTime", msgMetadata.getEventTime());
- traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
- traceDetails.put("uuid", msgMetadata.getUuid());
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
- if (entry == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateEntryDetails(level, entry, details);
- return details;
- }
-
- private static void populateEntryDetails(
- TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
- if (entry == null) {
- return;
- }
-
- switch (level) {
- case MINIMAL:
- traceDetails.put("ledgerId", entry.getLedgerId());
- traceDetails.put("entryId", entry.getEntryId());
- break;
- case BASIC:
- populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
-
- traceDetails.put("length", entry.getLength());
- break;
- case FULL:
- populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
- populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
-
- traceByteBuf("data", entry.getDataBuffer(), traceDetails);
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
- }
-
- public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
- if (publishContext == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populatePublishContext(publishContext, details);
- return details;
- }
-
- private static void populatePublishContext(
- Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
- traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
- traceDetails.put("isChunked", publishContext.isChunked());
- traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
-
- traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
- traceDetails.put("msgSize", publishContext.getMsgSize());
- traceDetails.put("producerName", publishContext.getProducerName());
- traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
- traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
- traceDetails.put("sequenceId", publishContext.getSequenceId());
- }
-
- private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
- if (buf == null) return;
-
- if (buf.readableBytes() < MAX_DATA_LENGTH) {
- traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
- } else {
- traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
- }
- }
-
private static TraceLevel getTracingLevel(Subscription sub) {
if (sub == null
|| sub.getSubscriptionProperties() == null
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
new file mode 100644
index 00000000..c875e27e
--- /dev/null
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -0,0 +1,361 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastax.oss.pulsar.jms.tracing;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import io.netty.buffer.ByteBuf;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.broker.service.Producer;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.Subscription;
+import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.naming.TopicName;
+
+@Slf4j
+public class TracingUtils {
+ private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+ private static final ObjectMapper mapper =
+ new ObjectMapper()
+ .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
+ .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
+
+ public static final int MAX_DATA_LENGTH = 1024;
+
+ public enum TraceLevel {
+ NONE,
+ MINIMAL,
+ BASIC,
+ FULL
+ }
+
+ public static void trace(String message, Map<String, Object> traceDetails) {
+ Map<String, Object> trace = new TreeMap<>();
+ trace.put("message", message);
+ trace.put("details", traceDetails);
+
+ try {
+ String loggableJsonString = mapper.writeValueAsString(trace);
+ tracer.info(loggableJsonString);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+ if (cnx == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateConnectionDetails(level, cnx, details);
+ return details;
+ }
+
+ private static void populateConnectionDetails(
+ TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+ if (cnx == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("clientAddress", cnx.clientAddress().toString());
+ break;
+ case BASIC:
+ populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
+
+ traceDetails.put("clientVersion", cnx.getClientVersion());
+ traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+ break;
+ case FULL:
+ populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
+ populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
+
+ traceDetails.put("authRole", cnx.getAuthRole());
+ traceDetails.put("authMethod", cnx.getAuthMethod());
+ traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+ if (sub == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateSubscriptionDetails(level, sub, details);
+ return details;
+ }
+
+ private static void populateSubscriptionDetails(
+ TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+ if (sub == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("name", sub.getName());
+ traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("type", sub.getType().name());
+ break;
+ case BASIC:
+ populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
+
+ if (sub.getConsumers() != null) {
+ traceDetails.put("numberOfConsumers", sub.getConsumers().size());
+ traceDetails.put(
+ "namesOfConsumers",
+ sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
+ }
+
+ break;
+ case FULL:
+ populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
+ populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
+
+ traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+ if (consumer == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateConsumerDetails(level, consumer, details);
+ return details;
+ }
+
+ private static void populateConsumerDetails(
+ TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+ if (consumer == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("name", consumer.consumerName());
+ traceDetails.put("consumerId", consumer.consumerId());
+ if (consumer.getSubscription() != null) {
+ traceDetails.put("subscriptionName", consumer.getSubscription().getName());
+ traceDetails.put(
+ "topicName",
+ TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+ }
+ break;
+ case BASIC:
+ populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+
+ traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+ traceDetails.put("subType", consumer.subType().name());
+ traceDetails.put("clientAddress", consumer.getClientAddress());
+ break;
+ case FULL:
+ populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+ populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+
+ traceDetails.put("metadata", consumer.getMetadata());
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+ if (producer == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateProducerDetails(level, producer, details);
+ return details;
+ }
+
+ private static void populateProducerDetails(
+ TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+ if (producer == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("producerId", producer.getProducerId());
+ traceDetails.put("producerName", producer.getProducerName());
+ traceDetails.put("accessMode", producer.getAccessMode().name());
+ if (producer.getTopic() != null) {
+ traceDetails.put(
+ "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+ }
+ break;
+ case BASIC:
+ populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+
+ traceDetails.put("clientAddress", producer.getClientAddress());
+ break;
+ case FULL:
+ populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+ populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+
+ traceDetails.put("metadata", producer.getMetadata());
+ if (producer.getSchemaVersion() != null) {
+ traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
+ }
+ traceDetails.put("remoteCluster", producer.getRemoteCluster());
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getMessageMetadataDetails(
+ TraceLevel level, MessageMetadata msgMetadata) {
+ if (msgMetadata == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateMessageMetadataDetails(level, msgMetadata, details);
+ return details;
+ }
+
+ private static void populateMessageMetadataDetails(
+ TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+ if (msgMetadata == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+ traceDetails.put("producerName", msgMetadata.getProducerName());
+ traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+ break;
+ case BASIC:
+ populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
+
+ traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+ traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+ traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+ break;
+ case FULL:
+ populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
+ populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
+
+ traceDetails.put("publishTime", msgMetadata.getPublishTime());
+ traceDetails.put("eventTime", msgMetadata.getEventTime());
+ traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+ traceDetails.put("uuid", msgMetadata.getUuid());
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+ if (entry == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateEntryDetails(level, entry, details);
+ return details;
+ }
+
+ private static void populateEntryDetails(
+ TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+ if (entry == null) {
+ return;
+ }
+
+ switch (level) {
+ case MINIMAL:
+ traceDetails.put("ledgerId", entry.getLedgerId());
+ traceDetails.put("entryId", entry.getEntryId());
+ break;
+ case BASIC:
+ populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+
+ traceDetails.put("length", entry.getLength());
+ break;
+ case FULL:
+ populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+ populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+
+ traceByteBuf("data", entry.getDataBuffer(), traceDetails);
+ break;
+ default:
+ log.warn("Unknown tracing level: {}", level);
+ break;
+ }
+ }
+
+ public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+ if (publishContext == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populatePublishContext(publishContext, details);
+ return details;
+ }
+
+ private static void populatePublishContext(
+ Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+ traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+ traceDetails.put("isChunked", publishContext.isChunked());
+ traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+
+ traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
+ traceDetails.put("msgSize", publishContext.getMsgSize());
+ traceDetails.put("producerName", publishContext.getProducerName());
+ traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+ traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+ traceDetails.put("sequenceId", publishContext.getSequenceId());
+ }
+
+ public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
+ if (buf == null) return;
+
+ if (buf.readableBytes() < MAX_DATA_LENGTH) {
+ traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+ } else {
+ traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+ }
+ }
+}
From 2246bc44efc58596f8976f255f85e8cdf40cb8eb Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 16:42:36 -0700
Subject: [PATCH 05/29] more config caching
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 101 ++++++++++++------
1 file changed, 70 insertions(+), 31 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index d3b7f248..3e0b91b8 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -36,6 +36,7 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
@@ -57,10 +58,19 @@
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
+ public enum EventReasons {
+ ADMINISTRATIVE,
+ MESSAGE,
+ TRANSACTION,
+ SERVLET,
+ }
+
+ private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+
private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
CacheBuilder.newBuilder()
.maximumSize(10)
- .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+ .expireAfterWrite(1, TimeUnit.MINUTES)
.build(
new CacheLoader<PulsarService, Set<EventReasons>>() {
public Set<EventReasons> load(PulsarService pulsar) {
@@ -89,7 +99,7 @@ public Set<EventReasons> load(PulsarService pulsar) {
private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
CacheBuilder.newBuilder()
.maximumSize(10)
- .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+ .expireAfterWrite(1, TimeUnit.MINUTES)
.build(
new CacheLoader<PulsarService, TraceLevel>() {
public TraceLevel load(PulsarService pulsar) {
@@ -108,14 +118,50 @@ public TraceLevel load(PulsarService pulsar) {
}
});
- public enum EventReasons {
- ADMINISTRATIVE,
- MESSAGE,
- TRANSACTION,
- SERVLET,
- }
+ private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
+ CacheBuilder.newBuilder()
+ .expireAfterWrite(10, TimeUnit.SECONDS)
+ .build(
+ new CacheLoader<Subscription, TraceLevel>() {
+ public TraceLevel load(Subscription sub) {
+ if (sub.getSubscriptionProperties() == null
+ || !sub.getSubscriptionProperties().containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
- private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+ try {
+ return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+ } catch (IllegalArgumentException e) {
+ log.warn(
+ "Invalid tracing level: {}. Setting to NONE for subscription {}",
+ sub.getSubscriptionProperties().get("trace"),
+ sub);
+ return TraceLevel.NONE;
+ }
+ }
+ });
+
+ private static final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
+ CacheBuilder.newBuilder()
+ .expireAfterWrite(10, TimeUnit.SECONDS)
+ .build(
+ new CacheLoader<Producer, TraceLevel>() {
+ public TraceLevel load(Producer producer) {
+ if (producer.getMetadata() == null
+ || !producer.getMetadata().containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
+ try {
+ return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+ } catch (IllegalArgumentException e) {
+ log.warn(
+ "Invalid tracing level: {}. Setting to NONE for producer {}",
+ producer.getMetadata().get("trace"),
+ producer);
+ return TraceLevel.NONE;
+ }
+ }
+ });
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
@@ -127,10 +173,14 @@ public void close() {
}
private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
+ if (cnx == null) return new HashSet<>();
+
return getEnabledEvents(cnx.getBrokerService().getPulsar());
}
private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
+ if (pulsar == null) return new HashSet<>();
+
try {
return jmsTracingEventList.get(pulsar);
} catch (ExecutionException e) {
@@ -140,6 +190,7 @@ private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
}
private static TraceLevel getTracingLevel(ServerCnx cnx) {
+ if (cnx == null) return defaultTraceLevel;
try {
return traceLevelForService.get(cnx.getBrokerService().getPulsar());
@@ -150,35 +201,23 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
}
private static TraceLevel getTracingLevel(Subscription sub) {
- if (sub == null
- || sub.getSubscriptionProperties() == null
- || !sub.getSubscriptionProperties().containsKey("trace")) {
- return TraceLevel.NONE;
- }
+ if (sub == null) return TraceLevel.NONE;
+
try {
- return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
- } catch (IllegalArgumentException e) {
- log.warn(
- "Invalid tracing level: {}. Setting to NONE for subscription {}",
- sub.getSubscriptionProperties().get("trace"),
- sub);
+ return traceLevelForSubscription.get(sub);
+ } catch (ExecutionException e) {
+ log.error("Error getting tracing level", e);
return TraceLevel.NONE;
}
}
private static TraceLevel getTracingLevel(Producer producer) {
- if (producer == null
- || producer.getMetadata() == null
- || !producer.getMetadata().containsKey("trace")) {
- return TraceLevel.NONE;
- }
+ if (producer == null) return TraceLevel.NONE;
+
try {
- return TraceLevel.valueOf(producer.getMetadata().get("trace"));
- } catch (IllegalArgumentException e) {
- log.warn(
- "Invalid tracing level: {}. Setting to NONE for producer {}",
- producer.getMetadata().get("trace"),
- producer);
+ return traceLevelForProducer.get(producer);
+ } catch (ExecutionException e) {
+ log.error("Error getting tracing level", e);
return TraceLevel.NONE;
}
}
From 4f84dc7dc795e09bfc8690caa8412b328dd47009 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 17:21:29 -0700
Subject: [PATCH 06/29] couple of tests
---
.../oss/pulsar/jms/tracing/TracingUtils.java | 22 +++-
.../pulsar/jms/tracing/TracingUtilsTest.java | 103 ++++++++++++++++++
2 files changed, 120 insertions(+), 5 deletions(-)
create mode 100644 pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index c875e27e..fbe4d88b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -35,7 +35,14 @@
@Slf4j
public class TracingUtils {
- private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+ private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+ @FunctionalInterface
+ public interface Tracer {
+ void trace(String message);
+ }
+
+ public static final Tracer SLF4J_TRACER = traceLogger::info;
private static final ObjectMapper mapper =
new ObjectMapper()
@@ -51,14 +58,19 @@ public enum TraceLevel {
FULL
}
+
public static void trace(String message, Map<String, Object> traceDetails) {
+ trace(SLF4J_TRACER, message, traceDetails);
+ }
+
+ public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
Map<String, Object> trace = new TreeMap<>();
trace.put("message", message);
- trace.put("details", traceDetails);
+ trace.put("traceDetails", traceDetails);
try {
String loggableJsonString = mapper.writeValueAsString(trace);
- tracer.info(loggableJsonString);
+ tracer.trace(loggableJsonString);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
@@ -353,9 +365,9 @@ public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> tra
if (buf == null) return;
if (buf.readableBytes() < MAX_DATA_LENGTH) {
- traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+ traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
} else {
- traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+ traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
}
}
}
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
new file mode 100644
index 00000000..956034cf
--- /dev/null
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -0,0 +1,103 @@
+package com.datastax.oss.pulsar.jms.tracing;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+
+class TracingUtilsTest {
+
+ List<String> traces = new ArrayList<>();
+ private Tracer mockTracer = new Tracer() {
+ @Override
+ public void trace(String msg) {
+ traces.add(msg);
+ }
+ };
+
+ @Test
+ void traceTest() {
+ traces.clear();
+ trace(mockTracer, "msg", null);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+
+ Map<String, Object> map = new TreeMap<>();
+
+ traces.clear();
+ trace(mockTracer, "msg", map);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+
+ map.put("key1", "value1");
+
+ traces.clear();
+ trace(mockTracer, "msg", map);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+ }
+
+// todo:
+// @Test
+// void getConnectionDetailsTest() {
+// }
+//
+// @Test
+// void getSubscriptionDetailsTest() {
+// }
+//
+// @Test
+// void getConsumerDetailsTest() {
+// }
+//
+// @Test
+// void getProducerDetails() {
+// }
+//
+// @Test
+// void getMessageMetadataDetailsTest() {
+// }
+//
+// @Test
+// void getEntryDetailsTest() {
+// }
+//
+// @Test
+// void getPublishContextDetailsTest() {
+// }
+
+ @Test
+ void traceByteBufTest() {
+ Map<String, Object> traceDetails = new TreeMap<>();
+
+ traceByteBuf("key", null, traceDetails);
+ assertEquals(0, traceDetails.size());
+
+ ByteBuf small = Unpooled.buffer(20);
+ for (int i = 0; i < 20; i++) {
+ small.writeByte(i);
+ }
+ traceByteBuf("key", small, traceDetails);
+ assertEquals(1, traceDetails.size());
+ assertEquals(42, ((String) traceDetails.get("key")).length());
+ assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+
+ ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
+ for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+ big.writeByte(i);
+ }
+
+ traceDetails.clear();
+ traceByteBuf("key", big, traceDetails);
+ assertEquals(1, traceDetails.size());
+ assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+ assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+ }
+}
\ No newline at end of file
From ffc604eeab31d91ecbca6563e026df8714669247 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 10:24:24 -0700
Subject: [PATCH 07/29] feedback
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 184 +++++++----------
.../oss/pulsar/jms/tracing/TracingUtils.java | 13 +-
.../pulsar/jms/tracing/TracingUtilsTest.java | 189 ++++++++++--------
3 files changed, 184 insertions(+), 202 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 3e0b91b8..a1d7062b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -54,6 +54,7 @@
import org.apache.pulsar.common.api.proto.CommandAck;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.intercept.InterceptException;
+import org.jetbrains.annotations.NotNull;
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
@@ -67,56 +68,40 @@ public enum EventReasons {
private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
- private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
- CacheBuilder.newBuilder()
- .maximumSize(10)
- .expireAfterWrite(1, TimeUnit.MINUTES)
- .build(
- new CacheLoader<PulsarService, Set<EventReasons>>() {
- public Set<EventReasons> load(PulsarService pulsar) {
- // dynamic config can get reloaded without service restart
- String events =
- pulsar
- .getConfiguration()
- .getProperties()
- .getProperty("jmsTracingEventList", "");
- log.debug("read jmsTracingEventList: {}", events);
-
- Set<EventReasons> enabledEvents = new HashSet<>();
-
- for (String event : events.split(",")) {
- try {
- enabledEvents.add(EventReasons.valueOf(event.trim()));
- } catch (IllegalArgumentException e) {
- log.warn("Invalid event: {}. Skipping", event);
- }
- }
+ private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
+ private TraceLevel traceLevel = defaultTraceLevel;
+
+ private static Set<EventReasons> loadEnabledEvents(
+ PulsarService pulsarService, Set<EventReasons> enabledEvents) {
+ String events =
+ pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
+ log.debug("read jmsTracingEventList: {}", events);
+
+ for (String event : events.split(",")) {
+ try {
+ enabledEvents.add(EventReasons.valueOf(event.trim()));
+ } catch (IllegalArgumentException e) {
+ log.error("Invalid event: {}. Skipping", event);
+ }
+ }
- return enabledEvents;
- }
- });
+ return enabledEvents;
+ }
- private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
- CacheBuilder.newBuilder()
- .maximumSize(10)
- .expireAfterWrite(1, TimeUnit.MINUTES)
- .build(
- new CacheLoader<PulsarService, TraceLevel>() {
- public TraceLevel load(PulsarService pulsar) {
- String level =
- pulsar
- .getConfiguration()
- .getProperties()
- .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
- try {
- return TraceLevel.valueOf(level);
- } catch (IllegalArgumentException e) {
- log.warn(
- "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
- return defaultTraceLevel;
- }
- }
- });
+ @NotNull
+ private static TraceLevel getTraceLevel(PulsarService pulsar) {
+ String level =
+ pulsar
+ .getConfiguration()
+ .getProperties()
+ .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+ try {
+ return TraceLevel.valueOf(level);
+ } catch (IllegalArgumentException e) {
+ log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+ return defaultTraceLevel;
+ }
+ }
private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
CacheBuilder.newBuilder()
@@ -124,17 +109,17 @@ public TraceLevel load(PulsarService pulsar) {
.build(
new CacheLoader<Subscription, TraceLevel>() {
public TraceLevel load(Subscription sub) {
- if (sub.getSubscriptionProperties() == null
- || !sub.getSubscriptionProperties().containsKey("trace")) {
+ Map<String, String> subProps = sub.getSubscriptionProperties();
+ if (subProps == null || !subProps.containsKey("trace")) {
return TraceLevel.NONE;
}
try {
- return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+ return TraceLevel.valueOf(subProps.get("trace"));
} catch (IllegalArgumentException e) {
log.warn(
"Invalid tracing level: {}. Setting to NONE for subscription {}",
- sub.getSubscriptionProperties().get("trace"),
+ subProps.get("trace"),
sub);
return TraceLevel.NONE;
}
@@ -165,6 +150,9 @@ public TraceLevel load(Producer producer) {
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
+
+ loadEnabledEvents(pulsarService, jmsTracingEventList);
+ traceLevel = getTraceLevel(pulsarService);
}
@Override
@@ -172,32 +160,9 @@ public void close() {
log.info("Closing BrokerTracing");
}
- private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
- if (cnx == null) return new HashSet<>();
-
- return getEnabledEvents(cnx.getBrokerService().getPulsar());
- }
-
- private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
- if (pulsar == null) return new HashSet<>();
-
- try {
- return jmsTracingEventList.get(pulsar);
- } catch (ExecutionException e) {
- log.error("Error getting enabled events", e);
- return new HashSet<>();
- }
- }
-
- private static TraceLevel getTracingLevel(ServerCnx cnx) {
- if (cnx == null) return defaultTraceLevel;
-
- try {
- return traceLevelForService.get(cnx.getBrokerService().getPulsar());
- } catch (ExecutionException e) {
- log.error("Error getting tracing level", e);
- return defaultTraceLevel;
- }
+ private static TraceLevel getTracingLevel(Consumer consumer) {
+ if (consumer == null) return TraceLevel.NONE;
+ return getTracingLevel(consumer.getSubscription());
}
private static TraceLevel getTracingLevel(Subscription sub) {
@@ -234,20 +199,19 @@ private static TraceLevel getTracingLevel(Producer producer) {
******************************/
public void onConnectionCreated(ServerCnx cnx) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
trace("Connection created", traceDetails);
}
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
+ TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -259,9 +223,9 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
}
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
+ TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -273,9 +237,9 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
}
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
+ TraceLevel level = getTracingLevel(consumer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -288,9 +252,9 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
}
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
+ TraceLevel level = getTracingLevel(consumer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -303,26 +267,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
}
public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
traceDetails.put("command", command.toString());
trace("Pulsar command called", traceDetails);
}
public void onConnectionClosed(ServerCnx cnx) {
- if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- TraceLevel level = getTracingLevel(cnx);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
trace("Connection closed", traceDetails);
}
@@ -337,8 +299,7 @@ public void beforeSendMessage(
long[] ackSet,
MessageMetadata msgMetadata,
Consumer consumer) {
- if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar())
- .contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(subscription);
if (level == TraceLevel.NONE) return;
@@ -355,8 +316,7 @@ public void beforeSendMessage(
public void onMessagePublish(
Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
- if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar())
- .contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.NONE) return;
@@ -376,7 +336,7 @@ public void messageProduced(
long ledgerId,
long entryId,
Topic.PublishContext publishContext) {
- if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.NONE) return;
@@ -394,9 +354,9 @@ public void messageProduced(
public void messageDispatched(
ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
- if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
- TraceLevel level = getTracingLevel(consumer.getSubscription());
+ TraceLevel level = getTracingLevel(consumer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -411,9 +371,9 @@ public void messageDispatched(
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
- if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
- TraceLevel level = getTracingLevel(consumer.getSubscription());
+ TraceLevel level = getTracingLevel(consumer);
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -442,23 +402,25 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
******************************/
public void txnOpened(long tcId, String txnID) {
- // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+ if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
+ if (traceLevel == TraceLevel.NONE) return;
+
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("tcId", tcId);
traceDetails.put("txnID", txnID);
trace("Transaction opened", traceDetails);
- // }
}
public void txnEnded(String txnID, long txnAction) {
- // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+ if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
+ if (traceLevel == TraceLevel.NONE) return;
+
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("txnID", txnID);
traceDetails.put("txnAction", txnAction);
trace("Transaction closed", traceDetails);
- // }
}
/* ***************************
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index fbe4d88b..bed9bf17 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -35,7 +35,8 @@
@Slf4j
public class TracingUtils {
- private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+ private static final org.slf4j.Logger traceLogger =
+ org.slf4j.LoggerFactory.getLogger("jms-tracing");
@FunctionalInterface
public interface Tracer {
@@ -58,7 +59,6 @@ public enum TraceLevel {
FULL
}
-
public static void trace(String message, Map<String, Object> traceDetails) {
trace(SLF4J_TRACER, message, traceDetails);
}
@@ -72,7 +72,11 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
String loggableJsonString = mapper.writeValueAsString(trace);
tracer.trace(loggableJsonString);
} catch (JsonProcessingException e) {
- throw new RuntimeException(e);
+ log.error(
+ "Failed to serialize trace message '{}' as json, traceDetails: {}",
+ message,
+ traceDetails,
+ e);
}
}
@@ -367,7 +371,8 @@ public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> tra
if (buf.readableBytes() < MAX_DATA_LENGTH) {
traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
} else {
- traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+ traceDetails.put(
+ key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
}
}
}
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 956034cf..4675de18 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -1,103 +1,118 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.datastax.oss.pulsar.jms.tracing;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+import static org.junit.jupiter.api.Assertions.*;
+
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
-import org.junit.jupiter.api.Test;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+import org.junit.jupiter.api.Test;
class TracingUtilsTest {
- List<String> traces = new ArrayList<>();
- private Tracer mockTracer = new Tracer() {
+ List<String> traces = new ArrayList<>();
+ private Tracer mockTracer =
+ new Tracer() {
@Override
public void trace(String msg) {
- traces.add(msg);
+ traces.add(msg);
}
- };
-
- @Test
- void traceTest() {
- traces.clear();
- trace(mockTracer, "msg", null);
- assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
-
- Map<String, Object> map = new TreeMap<>();
-
- traces.clear();
- trace(mockTracer, "msg", map);
- assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
-
- map.put("key1", "value1");
-
- traces.clear();
- trace(mockTracer, "msg", map);
- assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+ };
+
+ @Test
+ void traceTest() {
+ traces.clear();
+ trace(mockTracer, "msg", null);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+
+ Map<String, Object> map = new TreeMap<>();
+
+ traces.clear();
+ trace(mockTracer, "msg", map);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+
+ map.put("key1", "value1");
+
+ traces.clear();
+ trace(mockTracer, "msg", map);
+ assertEquals(1, traces.size());
+ assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+ }
+
+ // todo:
+ // @Test
+ // void getConnectionDetailsTest() {
+ // }
+ //
+ // @Test
+ // void getSubscriptionDetailsTest() {
+ // }
+ //
+ // @Test
+ // void getConsumerDetailsTest() {
+ // }
+ //
+ // @Test
+ // void getProducerDetails() {
+ // }
+ //
+ // @Test
+ // void getMessageMetadataDetailsTest() {
+ // }
+ //
+ // @Test
+ // void getEntryDetailsTest() {
+ // }
+ //
+ // @Test
+ // void getPublishContextDetailsTest() {
+ // }
+
+ @Test
+ void traceByteBufTest() {
+ Map<String, Object> traceDetails = new TreeMap<>();
+
+ traceByteBuf("key", null, traceDetails);
+ assertEquals(0, traceDetails.size());
+
+ ByteBuf small = Unpooled.buffer(20);
+ for (int i = 0; i < 20; i++) {
+ small.writeByte(i);
}
-
-// todo:
-// @Test
-// void getConnectionDetailsTest() {
-// }
-//
-// @Test
-// void getSubscriptionDetailsTest() {
-// }
-//
-// @Test
-// void getConsumerDetailsTest() {
-// }
-//
-// @Test
-// void getProducerDetails() {
-// }
-//
-// @Test
-// void getMessageMetadataDetailsTest() {
-// }
-//
-// @Test
-// void getEntryDetailsTest() {
-// }
-//
-// @Test
-// void getPublishContextDetailsTest() {
-// }
-
- @Test
- void traceByteBufTest() {
- Map<String, Object> traceDetails = new TreeMap<>();
-
- traceByteBuf("key", null, traceDetails);
- assertEquals(0, traceDetails.size());
-
- ByteBuf small = Unpooled.buffer(20);
- for (int i = 0; i < 20; i++) {
- small.writeByte(i);
- }
- traceByteBuf("key", small, traceDetails);
- assertEquals(1, traceDetails.size());
- assertEquals(42, ((String) traceDetails.get("key")).length());
- assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
-
- ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
- for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
- big.writeByte(i);
- }
-
- traceDetails.clear();
- traceByteBuf("key", big, traceDetails);
- assertEquals(1, traceDetails.size());
- assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
- assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+ traceByteBuf("key", small, traceDetails);
+ assertEquals(1, traceDetails.size());
+ assertEquals(42, ((String) traceDetails.get("key")).length());
+ assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+
+ ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
+ for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+ big.writeByte(i);
}
-}
\ No newline at end of file
+
+ traceDetails.clear();
+ traceByteBuf("key", big, traceDetails);
+ assertEquals(1, traceDetails.size());
+ assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+ assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+ }
+}
From 4585bcab8266c7f3b337249dbc3e600bbaed6468 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 14:19:26 -0700
Subject: [PATCH 08/29] fixes, tweaks
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 12 ++--
.../oss/pulsar/jms/tracing/TracingUtils.java | 62 ++++++++++++-------
2 files changed, 48 insertions(+), 26 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index a1d7062b..2b94a263 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -79,7 +79,7 @@ private static Set<EventReasons> loadEnabledEvents(
for (String event : events.split(",")) {
try {
- enabledEvents.add(EventReasons.valueOf(event.trim()));
+ enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
} catch (IllegalArgumentException e) {
log.error("Invalid event: {}. Skipping", event);
}
@@ -96,7 +96,7 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
.getProperties()
.getProperty("jmsTracingLevel", defaultTraceLevel.toString());
try {
- return TraceLevel.valueOf(level);
+ return TraceLevel.valueOf(level.trim().toUpperCase());
} catch (IllegalArgumentException e) {
log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
return defaultTraceLevel;
@@ -115,7 +115,7 @@ public TraceLevel load(Subscription sub) {
}
try {
- return TraceLevel.valueOf(subProps.get("trace"));
+ return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
} catch (IllegalArgumentException e) {
log.warn(
"Invalid tracing level: {}. Setting to NONE for subscription {}",
@@ -137,7 +137,8 @@ public TraceLevel load(Producer producer) {
return TraceLevel.NONE;
}
try {
- return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+ return TraceLevel.valueOf(
+ producer.getMetadata().get("trace").trim().toUpperCase());
} catch (IllegalArgumentException e) {
log.warn(
"Invalid tracing level: {}. Setting to NONE for producer {}",
@@ -273,6 +274,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+ // todo: .toString() is not good enough
+ // {"message":"Pulsar command
+ // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}}
traceDetails.put("command", command.toString());
trace("Pulsar command called", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index bed9bf17..372e6ca9 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -107,12 +107,15 @@ private static void populateConnectionDetails(
traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
break;
case FULL:
- populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
traceDetails.put("authRole", cnx.getAuthRole());
traceDetails.put("authMethod", cnx.getAuthMethod());
- traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+ traceDetails.put(
+ "authMethodName",
+ cnx.getAuthenticationProvider() == null
+ ? "no provider"
+ : cnx.getAuthenticationProvider().getAuthMethodName());
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -154,7 +157,6 @@ private static void populateSubscriptionDetails(
break;
case FULL:
- populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
@@ -185,22 +187,21 @@ private static void populateConsumerDetails(
case MINIMAL:
traceDetails.put("name", consumer.consumerName());
traceDetails.put("consumerId", consumer.consumerId());
- if (consumer.getSubscription() != null) {
- traceDetails.put("subscriptionName", consumer.getSubscription().getName());
+ Subscription sub = consumer.getSubscription();
+ if (sub != null) {
+ traceDetails.put("subscriptionName", sub.getName());
traceDetails.put(
- "topicName",
- TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+ "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
}
break;
case BASIC:
populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
traceDetails.put("priorityLevel", consumer.getPriorityLevel());
- traceDetails.put("subType", consumer.subType().name());
+ traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
traceDetails.put("clientAddress", consumer.getClientAddress());
break;
case FULL:
- populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
traceDetails.put("metadata", consumer.getMetadata());
@@ -231,7 +232,9 @@ private static void populateProducerDetails(
case MINIMAL:
traceDetails.put("producerId", producer.getProducerId());
traceDetails.put("producerName", producer.getProducerName());
- traceDetails.put("accessMode", producer.getAccessMode().name());
+ traceDetails.put(
+ "accessMode",
+ producer.getAccessMode() == null ? null : producer.getAccessMode().name());
if (producer.getTopic() != null) {
traceDetails.put(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
@@ -243,7 +246,6 @@ private static void populateProducerDetails(
traceDetails.put("clientAddress", producer.getClientAddress());
break;
case FULL:
- populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
traceDetails.put("metadata", producer.getMetadata());
@@ -277,25 +279,42 @@ private static void populateMessageMetadataDetails(
switch (level) {
case MINIMAL:
- traceDetails.put("sequenceId", msgMetadata.getSequenceId());
- traceDetails.put("producerName", msgMetadata.getProducerName());
- traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+ if (msgMetadata.hasPartitionKey()) {
+ traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+ }
+ if (msgMetadata.hasSequenceId()) {
+ traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+ }
+ if (msgMetadata.hasProducerName()) {
+ traceDetails.put("producerName", msgMetadata.getProducerName());
+ }
break;
case BASIC:
populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
- traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+ if (msgMetadata.hasUncompressedSize()) {
+ traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+ }
+ if (msgMetadata.hasNumMessagesInBatch()) {
+ traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+ }
traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
- traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
break;
case FULL:
- populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
- traceDetails.put("publishTime", msgMetadata.getPublishTime());
- traceDetails.put("eventTime", msgMetadata.getEventTime());
- traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
- traceDetails.put("uuid", msgMetadata.getUuid());
+ if (msgMetadata.hasPublishTime()) {
+ traceDetails.put("publishTime", msgMetadata.getPublishTime());
+ }
+ if (msgMetadata.hasEventTime()) {
+ traceDetails.put("eventTime", msgMetadata.getEventTime());
+ }
+ if (msgMetadata.hasReplicatedFrom()) {
+ traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+ }
+ if (msgMetadata.hasUuid()) {
+ traceDetails.put("uuid", msgMetadata.getUuid());
+ }
break;
default:
log.warn("Unknown tracing level: {}", level);
@@ -330,7 +349,6 @@ private static void populateEntryDetails(
traceDetails.put("length", entry.getLength());
break;
case FULL:
- populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
traceByteBuf("data", entry.getDataBuffer(), traceDetails);
From 5e9094a10a027e2d25711acb8c958cc2c2d0b169 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 16:20:37 -0700
Subject: [PATCH 09/29] actually trace commands; 'message' changed to
'eventType'
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 12 +-
.../oss/pulsar/jms/tracing/TracingUtils.java | 301 +++++++++++++++++-
2 files changed, 307 insertions(+), 6 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 2b94a263..7606563f 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -16,6 +16,7 @@
package com.datastax.oss.pulsar.jms.tracing;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
@@ -274,10 +275,13 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
- // todo: .toString() is not good enough
- // {"message":"Pulsar command
- // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}}
- traceDetails.put("command", command.toString());
+
+ if (command.hasType()) {
+ traceDetails.put("type", command.getType().name());
+ traceDetails.put("command", getCommandDetails(traceLevel, command));
+ } else {
+ traceDetails.put("type", "unknown/null");
+ }
trace("Pulsar command called", traceDetails);
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 372e6ca9..cc6ba6de 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -19,7 +19,11 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import io.netty.buffer.ByteBuf;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Map;
+import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@@ -30,6 +34,7 @@
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.Subscription;
import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.naming.TopicName;
@@ -65,7 +70,7 @@ public static void trace(String message, Map<String, Object> traceDetails) {
public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
Map<String, Object> trace = new TreeMap<>();
- trace.put("message", message);
+ trace.put("eventType", message);
trace.put("traceDetails", traceDetails);
try {
@@ -73,13 +78,293 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
tracer.trace(loggableJsonString);
} catch (JsonProcessingException e) {
log.error(
- "Failed to serialize trace message '{}' as json, traceDetails: {}",
+ "Failed to serialize trace event type '{}' as json, traceDetails: {}",
message,
traceDetails,
e);
}
}
+ public static Map<String, Object> getCommandDetails(TraceLevel level, BaseCommand command) {
+ if (command == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateCommandDetails(level, command, details);
+ return details;
+ }
+
+ private static void populateCommandDetails(
+ TraceLevel level, BaseCommand command, Map<String, Object> traceDetails) {
+ if (command == null) {
+ return;
+ }
+ if (level == TraceLevel.NONE) {
+ return;
+ }
+
+ if (!command.hasType()) {
+ return;
+ }
+
+ // trace all params otherwise
+
+ switch (command.getType()) {
+ case CONNECT:
+ populateByReflection(command.getConnect(), traceDetails);
+ break;
+ case CONNECTED:
+ populateByReflection(command.getConnected(), traceDetails);
+ break;
+ case SUBSCRIBE:
+ populateByReflection(command.getSubscribe(), traceDetails);
+ break;
+ case PRODUCER:
+ populateByReflection(command.getProducer(), traceDetails);
+ break;
+ case SEND:
+ populateByReflection(command.getSend(), traceDetails);
+ break;
+ case SEND_RECEIPT:
+ populateByReflection(command.getSendReceipt(), traceDetails);
+ break;
+ case SEND_ERROR:
+ populateByReflection(command.getSendError(), traceDetails);
+ break;
+ case MESSAGE:
+ populateByReflection(command.getMessage(), traceDetails);
+ break;
+ case ACK:
+ populateByReflection(command.getAck(), traceDetails);
+ break;
+ case FLOW:
+ populateByReflection(command.getFlow(), traceDetails);
+ break;
+ case UNSUBSCRIBE:
+ populateByReflection(command.getUnsubscribe(), traceDetails);
+ break;
+ case SUCCESS:
+ populateByReflection(command.getSuccess(), traceDetails);
+ break;
+ case ERROR:
+ populateByReflection(command.getError(), traceDetails);
+ break;
+ case CLOSE_PRODUCER:
+ populateByReflection(command.getCloseProducer(), traceDetails);
+ break;
+ case CLOSE_CONSUMER:
+ populateByReflection(command.getCloseConsumer(), traceDetails);
+ break;
+ case PRODUCER_SUCCESS:
+ populateByReflection(command.getProducerSuccess(), traceDetails);
+ break;
+ case PING:
+ populateByReflection(command.getPing(), traceDetails);
+ break;
+ case PONG:
+ populateByReflection(command.getPong(), traceDetails);
+ break;
+ case REDELIVER_UNACKNOWLEDGED_MESSAGES:
+ populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
+ break;
+ case PARTITIONED_METADATA:
+ populateByReflection(command.getPartitionMetadata(), traceDetails);
+ break;
+ case PARTITIONED_METADATA_RESPONSE:
+ populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
+ break;
+ case LOOKUP:
+ populateByReflection(command.getLookupTopic(), traceDetails);
+ break;
+ case LOOKUP_RESPONSE:
+ populateByReflection(command.getLookupTopicResponse(), traceDetails);
+ break;
+ case CONSUMER_STATS:
+ populateByReflection(command.getConsumerStats(), traceDetails);
+ break;
+ case CONSUMER_STATS_RESPONSE:
+ populateByReflection(command.getConsumerStatsResponse(), traceDetails);
+ break;
+ case REACHED_END_OF_TOPIC:
+ populateByReflection(command.getReachedEndOfTopic(), traceDetails);
+ break;
+ case SEEK:
+ populateByReflection(command.getSeek(), traceDetails);
+ break;
+ case GET_LAST_MESSAGE_ID:
+ populateByReflection(command.getGetLastMessageId(), traceDetails);
+ break;
+ case GET_LAST_MESSAGE_ID_RESPONSE:
+ populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
+ break;
+ case ACTIVE_CONSUMER_CHANGE:
+ populateByReflection(command.getActiveConsumerChange(), traceDetails);
+ break;
+ case GET_TOPICS_OF_NAMESPACE:
+ populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
+ break;
+ case GET_TOPICS_OF_NAMESPACE_RESPONSE:
+ populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
+ break;
+ case GET_SCHEMA:
+ populateByReflection(command.getGetSchema(), traceDetails);
+ break;
+ case GET_SCHEMA_RESPONSE:
+ populateByReflection(command.getGetSchemaResponse(), traceDetails);
+ break;
+ case AUTH_CHALLENGE:
+ populateByReflection(command.getAuthChallenge(), traceDetails);
+ break;
+ case AUTH_RESPONSE:
+ populateByReflection(command.getAuthResponse(), traceDetails);
+ break;
+ case ACK_RESPONSE:
+ populateByReflection(command.getAckResponse(), traceDetails);
+ break;
+ case GET_OR_CREATE_SCHEMA:
+ populateByReflection(command.getGetOrCreateSchema(), traceDetails);
+ break;
+ case GET_OR_CREATE_SCHEMA_RESPONSE:
+ populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
+ break;
+ case NEW_TXN:
+ populateByReflection(command.getNewTxn(), traceDetails);
+ break;
+ case NEW_TXN_RESPONSE:
+ populateByReflection(command.getNewTxnResponse(), traceDetails);
+ break;
+ case ADD_PARTITION_TO_TXN:
+ populateByReflection(command.getAddPartitionToTxn(), traceDetails);
+ break;
+ case ADD_PARTITION_TO_TXN_RESPONSE:
+ populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
+ break;
+ case ADD_SUBSCRIPTION_TO_TXN:
+ populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
+ break;
+ case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
+ populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
+ break;
+ case END_TXN:
+ populateByReflection(command.getEndTxn(), traceDetails);
+ break;
+ case END_TXN_RESPONSE:
+ populateByReflection(command.getEndTxnResponse(), traceDetails);
+ break;
+ case END_TXN_ON_PARTITION:
+ populateByReflection(command.getEndTxnOnPartition(), traceDetails);
+ break;
+ case END_TXN_ON_PARTITION_RESPONSE:
+ populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
+ break;
+ case END_TXN_ON_SUBSCRIPTION:
+ populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
+ break;
+ case END_TXN_ON_SUBSCRIPTION_RESPONSE:
+ populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
+ break;
+ case TC_CLIENT_CONNECT_REQUEST:
+ populateByReflection(command.getTcClientConnectRequest(), traceDetails);
+ break;
+ case TC_CLIENT_CONNECT_RESPONSE:
+ populateByReflection(command.getTcClientConnectResponse(), traceDetails);
+ break;
+ case WATCH_TOPIC_LIST:
+ populateByReflection(command.getWatchTopicList(), traceDetails);
+ break;
+ case WATCH_TOPIC_LIST_SUCCESS:
+ populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
+ break;
+ case WATCH_TOPIC_UPDATE:
+ populateByReflection(command.getWatchTopicUpdate(), traceDetails);
+ break;
+ case WATCH_TOPIC_LIST_CLOSE:
+ populateByReflection(command.getWatchTopicListClose(), traceDetails);
+ break;
+ case TOPIC_MIGRATED:
+ populateByReflection(command.getTopicMigrated(), traceDetails);
+ break;
+ default:
+ log.error("Unknown command type: {}", command.getType());
+ traceDetails.put("error", "unknownCommandType " + command.getType());
+ }
+ }
+
+ private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
+ if (command == null) {
+ return;
+ }
+ if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) {
+ return;
+ }
+
+ Method[] allMethods = command.getClass().getMethods();
+
+ Arrays.stream(allMethods)
+ .filter(method -> method.getName().startsWith("has"))
+ .filter(
+ method -> {
+ try {
+ return (boolean) method.invoke(command);
+ } catch (Exception e) {
+ return false;
+ }
+ })
+ .forEach(
+ method -> {
+ String fieldName = method.getName().substring(3);
+ try {
+ Optional<Method> accessor =
+ Arrays.stream(allMethods)
+ .filter(
+ m ->
+ m.getName().equals("get" + fieldName)
+ || m.getName().equals("is" + fieldName))
+ .findFirst();
+ if (!accessor.isPresent()) {
+ log.warn(
+ "No accessor found for field (but has.. counterpart was found): {} of {}",
+ fieldName,
+ command.getClass().getCanonicalName());
+ return;
+ }
+ Object value = accessor.get().invoke(command);
+
+ if (value == null) return;
+
+ // skip logging of binary data
+ if (value instanceof byte[]
+ || value instanceof ByteBuf
+ || value instanceof ByteBuffer) {
+ int size = 0;
+ if (value instanceof byte[]) {
+ size = ((byte[]) value).length;
+ } else if (value instanceof ByteBuf) {
+ size = ((ByteBuf) value).readableBytes();
+ } else if (value instanceof ByteBuffer) {
+ size = ((ByteBuffer) value).remaining();
+ }
+ traceDetails.put(fieldName + "_size", size);
+ return;
+ }
+
+ if (value
+ .getClass()
+ .getCanonicalName()
+ .contains("org.apache.pulsar.common.api.proto")) {
+ Map<String, Object> details = new TreeMap<>();
+ populateByReflection(value, details);
+ traceDetails.put(fieldName, details);
+ } else {
+ traceDetails.put(fieldName, value);
+ }
+ } catch (Exception e) {
+ log.error("Failed to access field: {}", fieldName, e);
+ }
+ });
+ }
+
public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
if (cnx == null) {
return null;
@@ -117,6 +402,8 @@ private static void populateConnectionDetails(
? "no provider"
: cnx.getAuthenticationProvider().getAuthMethodName());
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
@@ -161,6 +448,8 @@ private static void populateSubscriptionDetails(
traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
@@ -206,6 +495,8 @@ private static void populateConsumerDetails(
traceDetails.put("metadata", consumer.getMetadata());
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
@@ -254,6 +545,8 @@ private static void populateProducerDetails(
}
traceDetails.put("remoteCluster", producer.getRemoteCluster());
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
@@ -316,6 +609,8 @@ private static void populateMessageMetadataDetails(
traceDetails.put("uuid", msgMetadata.getUuid());
}
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
@@ -353,6 +648,8 @@ private static void populateEntryDetails(
traceByteBuf("data", entry.getDataBuffer(), traceDetails);
break;
+ case NONE:
+ break;
default:
log.warn("Unknown tracing level: {}", level);
break;
From f43457ffad0deb17a87fbd190a5afd48888b9d28 Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 11:40:26 +0200
Subject: [PATCH 10/29] Fix test
---
.../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 4675de18..fa148cc8 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -42,21 +42,21 @@ void traceTest() {
traces.clear();
trace(mockTracer, "msg", null);
assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+ assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
Map<String, Object> map = new TreeMap<>();
traces.clear();
trace(mockTracer, "msg", map);
assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+ assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
map.put("key1", "value1");
traces.clear();
trace(mockTracer, "msg", map);
assertEquals(1, traces.size());
- assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+ assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
}
// todo:
From 1d5e7b87a23943ff8d360b60a23980c851073edc Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 11:45:31 +0200
Subject: [PATCH 11/29] add pulsar-jms-tracing to the release and upgrade the
Nifi-plugin
---
.github/workflows/release.yml | 2 +-
pulsar-jms-admin-ext/pom.xml | 2 +-
pulsar-jms-filters/pom.xml | 2 +-
pulsar-jms-integration-tests/pom.xml | 4 ++--
pulsar-jms-tracing/pom.xml | 2 +-
pulsar-jms/pom.xml | 8 ++++----
6 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fc9e9cb1..2095b43c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -20,6 +20,6 @@ jobs:
run: mvn -B package -DskipTests
- uses: ncipollo/release-action@v1
with:
- artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar"
+ artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar,pulsar-jms-tracing/target/pulsar-jms*.nar"
token: ${{ secrets.GITHUB_TOKEN }}
generateReleaseNotes: true
diff --git a/pulsar-jms-admin-ext/pom.xml b/pulsar-jms-admin-ext/pom.xml
index d4754733..698c0711 100644
--- a/pulsar-jms-admin-ext/pom.xml
+++ b/pulsar-jms-admin-ext/pom.xml
@@ -66,7 +66,7 @@
<plugin>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-maven-plugin</artifactId>
- <version>1.3.2</version>
+ <version>1.5.1</version>
<extensions>true</extensions>
<configuration>
<finalName>pulsar-jms-admin-${project.version}</finalName>
diff --git a/pulsar-jms-filters/pom.xml b/pulsar-jms-filters/pom.xml
index 37e22a58..e8e86c98 100644
--- a/pulsar-jms-filters/pom.xml
+++ b/pulsar-jms-filters/pom.xml
@@ -85,7 +85,7 @@
<plugin>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-maven-plugin</artifactId>
- <version>1.3.2</version>
+ <version>1.5.1</version>
<extensions>true</extensions>
<configuration>
<finalName>pulsar-jms-${project.version}</finalName>
diff --git a/pulsar-jms-integration-tests/pom.xml b/pulsar-jms-integration-tests/pom.xml
index 798d61a8..747bb627 100644
--- a/pulsar-jms-integration-tests/pom.xml
+++ b/pulsar-jms-integration-tests/pom.xml
@@ -103,8 +103,8 @@
<configuration>
<target>
<echo>copy filters</echo>
- <mkdir dir="${project.build.outputDirectory}/filters" />
- <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar" />
+ <mkdir dir="${project.build.outputDirectory}/filters"/>
+ <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar"/>
</target>
</configuration>
</execution>
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index da7d2dda..fc87d02a 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -69,7 +69,7 @@
<plugin>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-maven-plugin</artifactId>
- <version>1.3.2</version>
+ <version>1.5.1</version>
<extensions>true</extensions>
<configuration>
<finalName>pulsar-jms-tracing-${project.version}</finalName>
diff --git a/pulsar-jms/pom.xml b/pulsar-jms/pom.xml
index b715013d..adc3d577 100644
--- a/pulsar-jms/pom.xml
+++ b/pulsar-jms/pom.xml
@@ -128,10 +128,10 @@
<configuration>
<target>
<echo>copy filters</echo>
- <mkdir dir="${project.build.outputDirectory}/filters" />
- <mkdir dir="${project.build.outputDirectory}/interceptors" />
- <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar" />
- <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/interceptors/jms-filter.nar" />
+ <mkdir dir="${project.build.outputDirectory}/filters"/>
+ <mkdir dir="${project.build.outputDirectory}/interceptors"/>
+ <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar"/>
+ <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/interceptors/jms-filter.nar"/>
</target>
</configuration>
</execution>
From 7c8750336fe46ebc7e5e89ac84c2e4f25c3ebdcb Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 16:31:24 +0200
Subject: [PATCH 12/29] fix build
---
pulsar-jms-tracing/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index fc87d02a..4c8ff528 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>pulsar-jms-parent</artifactId>
<groupId>com.datastax.oss</groupId>
- <version>4.1.1-alpha-SNAPSHOT</version>
+ <version>4.1.2-alpha-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pulsar-jms-tracing</artifactId>
From 58f11836c4f06f80f80f5a7ab003605bea0c28c8 Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 16:48:49 +0200
Subject: [PATCH 13/29] fix build
---
pulsar-jms-tracing/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index 4c8ff528..ed2bf109 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>pulsar-jms-parent</artifactId>
<groupId>com.datastax.oss</groupId>
- <version>4.1.2-alpha-SNAPSHOT</version>
+ <version>4.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pulsar-jms-tracing</artifactId>
From cbe3a5b034e037deaa380f1002a3323c7ccb91c0 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 16 Apr 2024 09:21:31 -0700
Subject: [PATCH 14/29] limit command output, separate event reason option for
commands
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 7 +-
.../oss/pulsar/jms/tracing/TracingUtils.java | 143 ++++++++++--------
2 files changed, 86 insertions(+), 64 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 7606563f..a4c44f98 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -62,6 +62,7 @@ public class BrokerTracing implements BrokerInterceptor {
public enum EventReasons {
ADMINISTRATIVE,
+ COMMANDS,
MESSAGE,
TRANSACTION,
SERVLET,
@@ -269,7 +270,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
}
public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
if (traceLevel == TraceLevel.NONE) return;
@@ -278,7 +279,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
if (command.hasType()) {
traceDetails.put("type", command.getType().name());
- traceDetails.put("command", getCommandDetails(traceLevel, command));
+ if (traceLevel != TraceLevel.MINIMAL) {
+ traceDetails.put("command", getCommandDetails(traceLevel, command));
+ }
} else {
traceDetails.put("type", "unknown/null");
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index cc6ba6de..86334ad6 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -18,12 +18,14 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@@ -109,181 +111,180 @@ private static void populateCommandDetails(
}
// trace all params otherwise
-
switch (command.getType()) {
case CONNECT:
- populateByReflection(command.getConnect(), traceDetails);
+ populateByReflection(level, command.getConnect(), traceDetails);
break;
case CONNECTED:
- populateByReflection(command.getConnected(), traceDetails);
+ populateByReflection(level, command.getConnected(), traceDetails);
break;
case SUBSCRIBE:
- populateByReflection(command.getSubscribe(), traceDetails);
+ populateByReflection(level, command.getSubscribe(), traceDetails);
break;
case PRODUCER:
- populateByReflection(command.getProducer(), traceDetails);
+ populateByReflection(level, command.getProducer(), traceDetails);
break;
case SEND:
- populateByReflection(command.getSend(), traceDetails);
+ populateByReflection(level, command.getSend(), traceDetails);
break;
case SEND_RECEIPT:
- populateByReflection(command.getSendReceipt(), traceDetails);
+ populateByReflection(level, command.getSendReceipt(), traceDetails);
break;
case SEND_ERROR:
- populateByReflection(command.getSendError(), traceDetails);
+ populateByReflection(level, command.getSendError(), traceDetails);
break;
case MESSAGE:
- populateByReflection(command.getMessage(), traceDetails);
+ populateByReflection(level, command.getMessage(), traceDetails);
break;
case ACK:
- populateByReflection(command.getAck(), traceDetails);
+ populateByReflection(level, command.getAck(), traceDetails);
break;
case FLOW:
- populateByReflection(command.getFlow(), traceDetails);
+ populateByReflection(level, command.getFlow(), traceDetails);
break;
case UNSUBSCRIBE:
- populateByReflection(command.getUnsubscribe(), traceDetails);
+ populateByReflection(level, command.getUnsubscribe(), traceDetails);
break;
case SUCCESS:
- populateByReflection(command.getSuccess(), traceDetails);
+ populateByReflection(level, command.getSuccess(), traceDetails);
break;
case ERROR:
- populateByReflection(command.getError(), traceDetails);
+ populateByReflection(level, command.getError(), traceDetails);
break;
case CLOSE_PRODUCER:
- populateByReflection(command.getCloseProducer(), traceDetails);
+ populateByReflection(level, command.getCloseProducer(), traceDetails);
break;
case CLOSE_CONSUMER:
- populateByReflection(command.getCloseConsumer(), traceDetails);
+ populateByReflection(level, command.getCloseConsumer(), traceDetails);
break;
case PRODUCER_SUCCESS:
- populateByReflection(command.getProducerSuccess(), traceDetails);
+ populateByReflection(level, command.getProducerSuccess(), traceDetails);
break;
case PING:
- populateByReflection(command.getPing(), traceDetails);
+ populateByReflection(level, command.getPing(), traceDetails);
break;
case PONG:
- populateByReflection(command.getPong(), traceDetails);
+ populateByReflection(level, command.getPong(), traceDetails);
break;
case REDELIVER_UNACKNOWLEDGED_MESSAGES:
- populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
+ populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails);
break;
case PARTITIONED_METADATA:
- populateByReflection(command.getPartitionMetadata(), traceDetails);
+ populateByReflection(level, command.getPartitionMetadata(), traceDetails);
break;
case PARTITIONED_METADATA_RESPONSE:
- populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
+ populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails);
break;
case LOOKUP:
- populateByReflection(command.getLookupTopic(), traceDetails);
+ populateByReflection(level, command.getLookupTopic(), traceDetails);
break;
case LOOKUP_RESPONSE:
- populateByReflection(command.getLookupTopicResponse(), traceDetails);
+ populateByReflection(level, command.getLookupTopicResponse(), traceDetails);
break;
case CONSUMER_STATS:
- populateByReflection(command.getConsumerStats(), traceDetails);
+ populateByReflection(level, command.getConsumerStats(), traceDetails);
break;
case CONSUMER_STATS_RESPONSE:
- populateByReflection(command.getConsumerStatsResponse(), traceDetails);
+ populateByReflection(level, command.getConsumerStatsResponse(), traceDetails);
break;
case REACHED_END_OF_TOPIC:
- populateByReflection(command.getReachedEndOfTopic(), traceDetails);
+ populateByReflection(level, command.getReachedEndOfTopic(), traceDetails);
break;
case SEEK:
- populateByReflection(command.getSeek(), traceDetails);
+ populateByReflection(level, command.getSeek(), traceDetails);
break;
case GET_LAST_MESSAGE_ID:
- populateByReflection(command.getGetLastMessageId(), traceDetails);
+ populateByReflection(level, command.getGetLastMessageId(), traceDetails);
break;
case GET_LAST_MESSAGE_ID_RESPONSE:
- populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
+ populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails);
break;
case ACTIVE_CONSUMER_CHANGE:
- populateByReflection(command.getActiveConsumerChange(), traceDetails);
+ populateByReflection(level, command.getActiveConsumerChange(), traceDetails);
break;
case GET_TOPICS_OF_NAMESPACE:
- populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
+ populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails);
break;
case GET_TOPICS_OF_NAMESPACE_RESPONSE:
- populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
+ populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails);
break;
case GET_SCHEMA:
- populateByReflection(command.getGetSchema(), traceDetails);
+ populateByReflection(level, command.getGetSchema(), traceDetails);
break;
case GET_SCHEMA_RESPONSE:
- populateByReflection(command.getGetSchemaResponse(), traceDetails);
+ populateByReflection(level, command.getGetSchemaResponse(), traceDetails);
break;
case AUTH_CHALLENGE:
- populateByReflection(command.getAuthChallenge(), traceDetails);
+ populateByReflection(level, command.getAuthChallenge(), traceDetails);
break;
case AUTH_RESPONSE:
- populateByReflection(command.getAuthResponse(), traceDetails);
+ populateByReflection(level, command.getAuthResponse(), traceDetails);
break;
case ACK_RESPONSE:
- populateByReflection(command.getAckResponse(), traceDetails);
+ populateByReflection(level, command.getAckResponse(), traceDetails);
break;
case GET_OR_CREATE_SCHEMA:
- populateByReflection(command.getGetOrCreateSchema(), traceDetails);
+ populateByReflection(level, command.getGetOrCreateSchema(), traceDetails);
break;
case GET_OR_CREATE_SCHEMA_RESPONSE:
- populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
+ populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails);
break;
case NEW_TXN:
- populateByReflection(command.getNewTxn(), traceDetails);
+ populateByReflection(level, command.getNewTxn(), traceDetails);
break;
case NEW_TXN_RESPONSE:
- populateByReflection(command.getNewTxnResponse(), traceDetails);
+ populateByReflection(level, command.getNewTxnResponse(), traceDetails);
break;
case ADD_PARTITION_TO_TXN:
- populateByReflection(command.getAddPartitionToTxn(), traceDetails);
+ populateByReflection(level, command.getAddPartitionToTxn(), traceDetails);
break;
case ADD_PARTITION_TO_TXN_RESPONSE:
- populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
+ populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails);
break;
case ADD_SUBSCRIPTION_TO_TXN:
- populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
+ populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails);
break;
case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
- populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
+ populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails);
break;
case END_TXN:
- populateByReflection(command.getEndTxn(), traceDetails);
+ populateByReflection(level, command.getEndTxn(), traceDetails);
break;
case END_TXN_RESPONSE:
- populateByReflection(command.getEndTxnResponse(), traceDetails);
+ populateByReflection(level, command.getEndTxnResponse(), traceDetails);
break;
case END_TXN_ON_PARTITION:
- populateByReflection(command.getEndTxnOnPartition(), traceDetails);
+ populateByReflection(level, command.getEndTxnOnPartition(), traceDetails);
break;
case END_TXN_ON_PARTITION_RESPONSE:
- populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
+ populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails);
break;
case END_TXN_ON_SUBSCRIPTION:
- populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
+ populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails);
break;
case END_TXN_ON_SUBSCRIPTION_RESPONSE:
- populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
+ populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails);
break;
case TC_CLIENT_CONNECT_REQUEST:
- populateByReflection(command.getTcClientConnectRequest(), traceDetails);
+ populateByReflection(level, command.getTcClientConnectRequest(), traceDetails);
break;
case TC_CLIENT_CONNECT_RESPONSE:
- populateByReflection(command.getTcClientConnectResponse(), traceDetails);
+ populateByReflection(level, command.getTcClientConnectResponse(), traceDetails);
break;
case WATCH_TOPIC_LIST:
- populateByReflection(command.getWatchTopicList(), traceDetails);
+ populateByReflection(level, command.getWatchTopicList(), traceDetails);
break;
case WATCH_TOPIC_LIST_SUCCESS:
- populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
+ populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails);
break;
case WATCH_TOPIC_UPDATE:
- populateByReflection(command.getWatchTopicUpdate(), traceDetails);
+ populateByReflection(level, command.getWatchTopicUpdate(), traceDetails);
break;
case WATCH_TOPIC_LIST_CLOSE:
- populateByReflection(command.getWatchTopicListClose(), traceDetails);
+ populateByReflection(level, command.getWatchTopicListClose(), traceDetails);
break;
case TOPIC_MIGRATED:
- populateByReflection(command.getTopicMigrated(), traceDetails);
+ populateByReflection(level, command.getTopicMigrated(), traceDetails);
break;
default:
log.error("Unknown command type: {}", command.getType());
@@ -291,7 +292,18 @@ private static void populateCommandDetails(
}
}
- private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
+ private static final Set<String> fullTraceFields =
+ Sets.newHashSet(
+ "authdata",
+ "authmethod",
+ "authmethodname",
+ "originalauthdata",
+ "orginalauthmethod",
+ "originalprincipal",
+ "schema");
+
+ private static void populateByReflection(
+ TraceLevel level, Object command, Map<String, Object> traceDetails) {
if (command == null) {
return;
}
@@ -302,7 +314,14 @@ private static void populateByReflection(Object command, Map<String, Object> tra
Method[] allMethods = command.getClass().getMethods();
Arrays.stream(allMethods)
- .filter(method -> method.getName().startsWith("has"))
+ .filter(
+ method -> {
+ if (!method.getName().startsWith("has")) {
+ return false;
+ }
+ String fieldName = method.getName().substring(3);
+ return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase());
+ })
.filter(
method -> {
try {
@@ -354,7 +373,7 @@ private static void populateByReflection(Object command, Map<String, Object> tra
.getCanonicalName()
.contains("org.apache.pulsar.common.api.proto")) {
Map<String, Object> details = new TreeMap<>();
- populateByReflection(value, details);
+ populateByReflection(level, value, details);
traceDetails.put(fieldName, details);
} else {
traceDetails.put(fieldName, value);
From 7e480aadcf33b17a29bb1631a4c98e70e0f0c5c5 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 12:04:44 -0700
Subject: [PATCH 15/29] more configuration parameters to reduce verbosity
---
.../oss/pulsar/jms/selectors/JMSFilter.java | 3 +-
.../oss/pulsar/jms/tracing/BrokerTracing.java | 116 +++++++++++++-----
.../oss/pulsar/jms/tracing/TracingUtils.java | 52 ++++----
.../pulsar/jms/tracing/TracingUtilsTest.java | 14 ++-
4 files changed, 126 insertions(+), 59 deletions(-)
diff --git a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
index dc97f9f5..f8b22834 100644
--- a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
+++ b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
@@ -68,7 +68,8 @@ public class JMSFilter implements EntryFilter {
try {
CollectorRegistry.defaultRegistry.register(filterProcessingTime);
} catch (IllegalArgumentException alreadyRegistered) {
- // this happens in Pulsar 2.10, because each JMSFilter is created in a different classloader
+ // this happens in Pulsar 2.10, because each JMSFilter is created in a different
+ // classloader
// so the metricRegistered flag doesn't help
log.debug("JMSFilter metrics already registered");
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index a4c44f98..7a331e69 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
@@ -72,6 +73,10 @@ public enum EventReasons {
private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
private TraceLevel traceLevel = defaultTraceLevel;
+ private int maxBinaryDataLength = 256;
+ private boolean traceSystemTopics = false;
+ private boolean traceSchema = false;
+ private boolean reduceLevelForNestedComponents = true;
private static Set<EventReasons> loadEnabledEvents(
PulsarService pulsarService, Set<EventReasons> enabledEvents) {
@@ -136,7 +141,7 @@ public TraceLevel load(Subscription sub) {
public TraceLevel load(Producer producer) {
if (producer.getMetadata() == null
|| !producer.getMetadata().containsKey("trace")) {
- return TraceLevel.NONE;
+ return TraceLevel.FULL;
}
try {
return TraceLevel.valueOf(
@@ -146,7 +151,7 @@ public TraceLevel load(Producer producer) {
"Invalid tracing level: {}. Setting to NONE for producer {}",
producer.getMetadata().get("trace"),
producer);
- return TraceLevel.NONE;
+ return TraceLevel.FULL;
}
}
});
@@ -156,6 +161,21 @@ public void initialize(PulsarService pulsarService) {
loadEnabledEvents(pulsarService, jmsTracingEventList);
traceLevel = getTraceLevel(pulsarService);
+
+ Properties props = pulsarService.getConfiguration().getProperties();
+ if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
+ maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
+ }
+ if (props.containsKey("jmsTracingTraceSystemTopics")) {
+ traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
+ }
+ if (props.containsKey("jmsTracingTraceSchema")) {
+ traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
+ }
+ if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) {
+ reduceLevelForNestedComponents =
+ Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
+ }
}
@Override
@@ -163,14 +183,17 @@ public void close() {
log.info("Closing BrokerTracing");
}
- private static TraceLevel getTracingLevel(Consumer consumer) {
+ private TraceLevel getTracingLevel(Consumer consumer) {
if (consumer == null) return TraceLevel.NONE;
+
return getTracingLevel(consumer.getSubscription());
}
- private static TraceLevel getTracingLevel(Subscription sub) {
+ private TraceLevel getTracingLevel(Subscription sub) {
if (sub == null) return TraceLevel.NONE;
+ if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE;
+
try {
return traceLevelForSubscription.get(sub);
} catch (ExecutionException e) {
@@ -179,9 +202,11 @@ private static TraceLevel getTracingLevel(Subscription sub) {
}
}
- private static TraceLevel getTracingLevel(Producer producer) {
+ private TraceLevel getTracingLevel(Producer producer) {
if (producer == null) return TraceLevel.NONE;
+ if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE;
+
try {
return traceLevelForProducer.get(producer);
} catch (ExecutionException e) {
@@ -190,6 +215,13 @@ private static TraceLevel getTracingLevel(Producer producer) {
}
}
+ private TraceLevel getTraceLevelForComponent(TraceLevel current) {
+ if (current == TraceLevel.NONE) return TraceLevel.NONE;
+ if (reduceLevelForNestedComponents) return TraceLevel.BASIC;
+
+ return current;
+ }
+
// private boolean needToTraceTopic(Topic topic) {
// if (topic == null) return false;
//
@@ -218,8 +250,8 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+ traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
traceDetails.put("metadata", metadata);
trace("Producer created", traceDetails);
@@ -232,8 +264,8 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+ traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
traceDetails.put("metadata", metadata);
trace("Producer closed", traceDetails);
@@ -246,7 +278,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
traceDetails.put("consumer", getConsumerDetails(level, consumer));
traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
traceDetails.put("metadata", metadata);
@@ -275,7 +307,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
if (command.hasType()) {
traceDetails.put("type", command.getType().name());
@@ -316,9 +348,10 @@ public void beforeSendMessage(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("subscription", getSubscriptionDetails(level, subscription));
- traceDetails.put("consumer", getConsumerDetails(level, consumer));
- traceDetails.put("entry", getEntryDetails(level, entry));
+ traceDetails.put(
+ "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription));
+ traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+ traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength));
traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
trace("Before sending message", traceDetails);
@@ -333,9 +366,10 @@ public void onMessagePublish(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put(
+ "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
- traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+ traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
trace("Message publish", traceDetails);
}
@@ -353,11 +387,11 @@ public void messageProduced(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("producer", getProducerDetails(level, producer));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+ traceDetails.put(
+ "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
- traceDetails.put("ledgerId", ledgerId);
- traceDetails.put("entryId", entryId);
+ traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
trace("Message produced", traceDetails);
@@ -371,12 +405,13 @@ public void messageDispatched(
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("consumer", getConsumerDetails(level, consumer));
- traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
- traceDetails.put("ledgerId", ledgerId);
- traceDetails.put("entryId", entryId);
- traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+ traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+ traceDetails.put(
+ "subscription",
+ getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+ traceDetails.put("messageId", ledgerId + ":" + entryId);
+ traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
trace("After dispatching message", traceDetails);
}
@@ -388,13 +423,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (level == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("consumer", getConsumerDetails(level, consumer));
- traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+ traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+ traceDetails.put(
+ "subscription",
+ getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
Map<String, Object> ackDetails = new TreeMap<>();
- ackDetails.put("type", ackCmd.getAckType().name());
- ackDetails.put("consumerId", ackCmd.getConsumerId());
+ if (ackCmd.hasAckType()) {
+ ackDetails.put("type", ackCmd.getAckType().name());
+ } else {
+ ackDetails.put("type", "NOT SET");
+ }
+ if (ackCmd.hasConsumerId()) {
+ ackDetails.put("consumerId", ackCmd.getConsumerId());
+ } else {
+ ackDetails.put("consumerId", "NOT SET");
+ }
+ ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
ackDetails.put(
"messageIds",
ackCmd
@@ -403,6 +449,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
.map(x -> x.getLedgerId() + ":" + x.getEntryId())
.collect(Collectors.toList()));
+ if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) {
+ ackDetails.put(
+ "txnID", "(" + ackCmd.getTxnidMostBits() + "," + ackCmd.getTxnidLeastBits() + ")");
+ }
+ if (ackCmd.hasRequestId()) {
+ ackDetails.put("requestId", ackCmd.getRequestId());
+ }
+
traceDetails.put("ack", ackDetails);
trace("Message acked", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 86334ad6..87c3ca13 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -39,6 +39,7 @@
import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.schema.SchemaVersion;
@Slf4j
public class TracingUtils {
@@ -57,8 +58,6 @@ public interface Tracer {
.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
.enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
- public static final int MAX_DATA_LENGTH = 1024;
-
public enum TraceLevel {
NONE,
MINIMAL,
@@ -522,18 +521,19 @@ private static void populateConsumerDetails(
}
}
- public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+ public static Map<String, Object> getProducerDetails(
+ TraceLevel level, Producer producer, boolean traceSchema) {
if (producer == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateProducerDetails(level, producer, details);
+ populateProducerDetails(level, producer, details, traceSchema);
return details;
}
private static void populateProducerDetails(
- TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+ TraceLevel level, Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
if (producer == null) {
return;
}
@@ -551,16 +551,25 @@ private static void populateProducerDetails(
}
break;
case BASIC:
- populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+ populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema);
traceDetails.put("clientAddress", producer.getClientAddress());
break;
case FULL:
- populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+ populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema);
traceDetails.put("metadata", producer.getMetadata());
- if (producer.getSchemaVersion() != null) {
- traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
+
+ if (traceSchema && producer.getSchemaVersion() != null) {
+ final String schemaVersion;
+ if (producer.getSchemaVersion() == SchemaVersion.Empty) {
+ schemaVersion = "Empty";
+ } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
+ schemaVersion = "Latest";
+ } else {
+ schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
+ }
+ traceDetails.put("schemaVersion", schemaVersion);
}
traceDetails.put("remoteCluster", producer.getRemoteCluster());
break;
@@ -636,36 +645,36 @@ private static void populateMessageMetadataDetails(
}
}
- public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+ public static Map<String, Object> getEntryDetails(
+ TraceLevel level, Entry entry, int maxBinaryDataLength) {
if (entry == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateEntryDetails(level, entry, details);
+ populateEntryDetails(level, entry, details, maxBinaryDataLength);
return details;
}
private static void populateEntryDetails(
- TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+ TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
if (entry == null) {
return;
}
switch (level) {
case MINIMAL:
- traceDetails.put("ledgerId", entry.getLedgerId());
- traceDetails.put("entryId", entry.getEntryId());
+ traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
break;
case BASIC:
- populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+ populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength);
traceDetails.put("length", entry.getLength());
break;
case FULL:
- populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+ populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength);
- traceByteBuf("data", entry.getDataBuffer(), traceDetails);
+ traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
break;
case NONE:
break;
@@ -699,14 +708,15 @@ private static void populatePublishContext(
traceDetails.put("sequenceId", publishContext.getSequenceId());
}
- public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
- if (buf == null) return;
+ public static void traceByteBuf(
+ String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+ if (buf == null || maxBinaryDataLength <= 0) return;
- if (buf.readableBytes() < MAX_DATA_LENGTH) {
+ if (buf.readableBytes() < maxBinaryDataLength) {
traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
} else {
traceDetails.put(
- key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+ key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()));
}
}
}
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index fa148cc8..220fee87 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -92,27 +92,29 @@ void traceTest() {
void traceByteBufTest() {
Map<String, Object> traceDetails = new TreeMap<>();
- traceByteBuf("key", null, traceDetails);
+ int maxBinaryDataLength = 1024;
+
+ traceByteBuf("key", null, traceDetails, maxBinaryDataLength);
assertEquals(0, traceDetails.size());
ByteBuf small = Unpooled.buffer(20);
for (int i = 0; i < 20; i++) {
small.writeByte(i);
}
- traceByteBuf("key", small, traceDetails);
+ traceByteBuf("key", small, traceDetails, maxBinaryDataLength);
assertEquals(1, traceDetails.size());
assertEquals(42, ((String) traceDetails.get("key")).length());
assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
- ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
- for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+ ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100);
+ for (int i = 0; i < maxBinaryDataLength + 100; i++) {
big.writeByte(i);
}
traceDetails.clear();
- traceByteBuf("key", big, traceDetails);
+ traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
assertEquals(1, traceDetails.size());
- assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+ assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length());
assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
}
}
From 0a42c1f50a818d74493e2fd6dc9c55b451847a11 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 14:43:02 -0700
Subject: [PATCH 16/29] updated version to match master / merged poms
---
pulsar-jms-tracing/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index ed2bf109..19b588df 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>pulsar-jms-parent</artifactId>
<groupId>com.datastax.oss</groupId>
- <version>4.1.2-SNAPSHOT</version>
+ <version>4.1.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>pulsar-jms-tracing</artifactId>
From 9c1ad66b0f8fc094876e4855a2c8f51ff98bf7c2 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 15:58:06 -0700
Subject: [PATCH 17/29] more message id details, when available
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 7a331e69..2ab6da2b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -54,6 +54,7 @@
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageIdData;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.intercept.InterceptException;
import org.jetbrains.annotations.NotNull;
@@ -393,7 +394,6 @@ public void messageProduced(
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
-
trace("Message produced", traceDetails);
}
@@ -446,7 +446,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
ackCmd
.getMessageIdsList()
.stream()
- .map(x -> x.getLedgerId() + ":" + x.getEntryId())
+ .map(BrokerTracing::formatMessageId)
.collect(Collectors.toList()));
if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) {
@@ -462,6 +462,17 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
trace("Message acked", traceDetails);
}
+ @NotNull
+ private static String formatMessageId(MessageIdData x) {
+ String msgId = x.getLedgerId() + ":" + x.getEntryId();
+ if (x.hasBatchIndex()) {
+ msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
+ } else if (x.getAckSetsCount() > 0) {
+ msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
+ }
+ return msgId;
+ }
+
/* ***************************
** Transaction events
******************************/
From a9d74f8a05e7272d60e9477759563cfee8dbb879 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 30 Apr 2024 15:44:43 -0700
Subject: [PATCH 18/29] Trace levels for producer/consumer can be set via topic
properties; a bit more infor for auth is traced
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++-------
.../oss/pulsar/jms/tracing/TracingUtils.java | 37 +++++-
2 files changed, 112 insertions(+), 46 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 2ab6da2b..98aa617b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -52,11 +52,13 @@
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.Subscription;
import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.CommandAck;
import org.apache.pulsar.common.api.proto.MessageIdData;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.naming.TopicName;
import org.jetbrains.annotations.NotNull;
@Slf4j
@@ -75,6 +77,7 @@ public enum EventReasons {
private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
private TraceLevel traceLevel = defaultTraceLevel;
private int maxBinaryDataLength = 256;
+ private int cacheTraceLevelsDurationSec = 10;
private boolean traceSystemTopics = false;
private boolean traceSchema = false;
private boolean reduceLevelForNestedComponents = true;
@@ -111,18 +114,20 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
}
}
- private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
+ private final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
CacheBuilder.newBuilder()
- .expireAfterWrite(10, TimeUnit.SECONDS)
+ .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
.build(
new CacheLoader<Subscription, TraceLevel>() {
public TraceLevel load(Subscription sub) {
Map<String, String> subProps = sub.getSubscriptionProperties();
- if (subProps == null || !subProps.containsKey("trace")) {
- return TraceLevel.NONE;
- }
-
try {
+ if (subProps == null || !subProps.containsKey("trace")) {
+ PulsarAdmin admin =
+ sub.getTopic().getBrokerService().getPulsar().getAdminClient();
+ return BrokerTracing.getTracingLevel(admin, sub.getTopic());
+ }
+
return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
} catch (IllegalArgumentException e) {
log.warn(
@@ -130,33 +135,55 @@ public TraceLevel load(Subscription sub) {
subProps.get("trace"),
sub);
return TraceLevel.NONE;
+ } catch (Throwable t) {
+ log.error("Error getting tracing level", t);
+ return TraceLevel.NONE;
}
}
});
- private static final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
+ private final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
CacheBuilder.newBuilder()
- .expireAfterWrite(10, TimeUnit.SECONDS)
+ .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
.build(
new CacheLoader<Producer, TraceLevel>() {
public TraceLevel load(Producer producer) {
- if (producer.getMetadata() == null
- || !producer.getMetadata().containsKey("trace")) {
- return TraceLevel.FULL;
- }
try {
- return TraceLevel.valueOf(
- producer.getMetadata().get("trace").trim().toUpperCase());
- } catch (IllegalArgumentException e) {
- log.warn(
- "Invalid tracing level: {}. Setting to NONE for producer {}",
- producer.getMetadata().get("trace"),
- producer);
- return TraceLevel.FULL;
+ PulsarAdmin admin =
+ producer.getCnx().getBrokerService().getPulsar().getAdminClient();
+ Topic topic = producer.getTopic();
+
+ return BrokerTracing.getTracingLevel(admin, topic);
+ } catch (Throwable t) {
+ log.error("Error getting tracing level", t);
+ return TraceLevel.NONE;
}
}
});
+ @NotNull
+ private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) {
+ Map<String, String> props = null;
+ try {
+ props =
+ admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName());
+
+ if (props == null || !props.containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
+
+ return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ log.warn(
+ "Invalid tracing level: {}. Setting to NONE",
+ props == null ? "no props" : props.get("trace"));
+ return TraceLevel.NONE;
+ } catch (Throwable t) {
+ log.error("Error getting tracing level", t);
+ return TraceLevel.NONE;
+ }
+ }
+
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
@@ -177,6 +204,15 @@ public void initialize(PulsarService pulsarService) {
reduceLevelForNestedComponents =
Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
}
+ if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
+ cacheTraceLevelsDurationSec =
+ Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec"));
+ if (cacheTraceLevelsDurationSec <= 0) {
+ log.warn(
+ "Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10);
+ cacheTraceLevelsDurationSec = 10;
+ }
+ }
}
@Override
@@ -223,13 +259,6 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) {
return current;
}
- // private boolean needToTraceTopic(Topic topic) {
- // if (topic == null) return false;
- //
- // // todo: how to read topic props
- // return true;
- // }
-
/* ***************************
** Administrative events
******************************/
@@ -246,13 +275,13 @@ public void onConnectionCreated(ServerCnx cnx) {
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- TraceLevel level = getTracingLevel(producer);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+ traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
traceDetails.put("metadata", metadata);
trace("Producer created", traceDetails);
@@ -260,13 +289,13 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- TraceLevel level = getTracingLevel(producer);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+ traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
traceDetails.put("metadata", metadata);
trace("Producer closed", traceDetails);
@@ -274,14 +303,15 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- TraceLevel level = getTracingLevel(consumer);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put("consumer", getConsumerDetails(level, consumer));
- traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+ traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
+ traceDetails.put(
+ "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
traceDetails.put("metadata", metadata);
trace("Consumer created", traceDetails);
@@ -289,14 +319,15 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- TraceLevel level = getTracingLevel(consumer);
- if (level == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.NONE) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
- traceDetails.put("consumer", getConsumerDetails(level, consumer));
- traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+ traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
+ traceDetails.put(
+ "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
traceDetails.put("metadata", metadata);
trace("Consumer closed", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 87c3ca13..4e6311df 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -31,6 +31,7 @@
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.service.Consumer;
import org.apache.pulsar.broker.service.Producer;
import org.apache.pulsar.broker.service.ServerCnx;
@@ -402,6 +403,7 @@ private static void populateConnectionDetails(
switch (level) {
case MINIMAL:
traceDetails.put("clientAddress", cnx.clientAddress().toString());
+ traceDetails.put("authRole", cnx.getAuthRole());
break;
case BASIC:
populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
@@ -412,13 +414,17 @@ private static void populateConnectionDetails(
case FULL:
populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
- traceDetails.put("authRole", cnx.getAuthRole());
traceDetails.put("authMethod", cnx.getAuthMethod());
traceDetails.put(
"authMethodName",
cnx.getAuthenticationProvider() == null
? "no provider"
: cnx.getAuthenticationProvider().getAuthMethodName());
+
+ AuthenticationDataSource authData = cnx.getAuthenticationData();
+ if (authData != null) {
+ traceDetails.put("authData", getAuthDataDetails(authData));
+ }
break;
case NONE:
break;
@@ -428,6 +434,35 @@ private static void populateConnectionDetails(
}
}
+ private static Object getAuthDataDetails(AuthenticationDataSource authData) {
+ if (authData == null) {
+ return null;
+ }
+
+ Map<String, Object> details = new TreeMap<>();
+ populateAuthDataDetails(authData, details);
+ return details;
+ }
+
+ private static void populateAuthDataDetails(
+ AuthenticationDataSource authData, Map<String, Object> details) {
+ if (authData == null) {
+ return;
+ }
+
+ details.put("peerAddress", authData.getPeerAddress());
+ details.put("commandData", authData.getCommandData());
+ details.put("httpAuthType", authData.getHttpAuthType());
+ details.put("subscription", authData.getSubscription());
+ if (authData.getTlsCertificates() != null) {
+ details.put(
+ "tlsCertificates",
+ Arrays.stream(authData.getTlsCertificates())
+ .map(Object::toString)
+ .collect(Collectors.toList()));
+ }
+ }
+
public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
if (sub == null) {
return null;
From e910fa6170de5cb2a553b17218d6a61d3d6270a8 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 1 May 2024 12:40:06 -0700
Subject: [PATCH 19/29] async refresh for caches
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 151 ++++++++++++++----
1 file changed, 118 insertions(+), 33 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 98aa617b..8a77abee 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -30,6 +30,8 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
import io.netty.buffer.ByteBuf;
import java.io.IOException;
import java.util.HashSet;
@@ -37,6 +39,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -116,74 +119,156 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
private final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
CacheBuilder.newBuilder()
- .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+ .maximumSize(10_000L)
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+ .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
.build(
new CacheLoader<Subscription, TraceLevel>() {
public TraceLevel load(Subscription sub) {
- Map<String, String> subProps = sub.getSubscriptionProperties();
- try {
- if (subProps == null || !subProps.containsKey("trace")) {
- PulsarAdmin admin =
- sub.getTopic().getBrokerService().getPulsar().getAdminClient();
- return BrokerTracing.getTracingLevel(admin, sub.getTopic());
- }
-
- return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
- } catch (IllegalArgumentException e) {
- log.warn(
- "Invalid tracing level: {}. Setting to NONE for subscription {}",
- subProps.get("trace"),
- sub);
- return TraceLevel.NONE;
- } catch (Throwable t) {
- log.error("Error getting tracing level", t);
- return TraceLevel.NONE;
- }
+ log.info("Loading trace level for subscription {}", sub);
+ return BrokerTracing.readTraceLevelForSubscription(sub);
}
- });
+ public ListenableFuture<TraceLevel> reload(Subscription sub, TraceLevel oldValue)
+ throws Exception {
+ SettableFuture<TraceLevel> future = SettableFuture.create();
+ BrokerTracing.readTraceLevelForSubscriptionAsync(sub)
+ .whenComplete(
+ (level, ex) -> {
+ if (ex != null) {
+ future.setException(ex);
+ } else {
+ future.set(level);
+ }
+ });
+ return future;
+ }
+ });
private final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
CacheBuilder.newBuilder()
- .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+ .maximumSize(10_000L)
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+ .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
.build(
new CacheLoader<Producer, TraceLevel>() {
public TraceLevel load(Producer producer) {
try {
+ log.info("Loading trace level for producer {}", producer);
PulsarAdmin admin =
producer.getCnx().getBrokerService().getPulsar().getAdminClient();
Topic topic = producer.getTopic();
- return BrokerTracing.getTracingLevel(admin, topic);
+ return BrokerTracing.readTraceLevelForTopic(admin, topic);
} catch (Throwable t) {
log.error("Error getting tracing level", t);
return TraceLevel.NONE;
}
}
+
+ public ListenableFuture<TraceLevel> reload(Producer producer, TraceLevel oldValue)
+ throws Exception {
+ SettableFuture<TraceLevel> future = SettableFuture.create();
+
+ PulsarAdmin admin =
+ producer.getCnx().getBrokerService().getPulsar().getAdminClient();
+ Topic topic = producer.getTopic();
+
+ BrokerTracing.readTraceLevelForTopicAsync(admin, topic)
+ .whenComplete(
+ (level, ex) -> {
+ if (ex != null) {
+ future.setException(ex);
+ } else {
+ future.set(level);
+ }
+ });
+
+ return future;
+ }
});
@NotNull
- private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) {
- Map<String, String> props = null;
+ private static TraceLevel readTraceLevelForSubscription(Subscription sub) {
try {
- props =
- admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName());
+ return readTraceLevelForSubscriptionAsync(sub).get();
+ } catch (InterruptedException | ExecutionException e) {
+ log.error("Interrupted while getting subscription tracing level for {}", sub, e);
+ Thread.currentThread().interrupt();
+ return TraceLevel.NONE;
+ } catch (Throwable t) {
+ log.error("Error getting subscription tracing level for {}", sub, t);
+ return TraceLevel.NONE;
+ }
+ }
- if (props == null || !props.containsKey("trace")) {
- return TraceLevel.NONE;
+ @NotNull
+ private static CompletableFuture<TraceLevel> readTraceLevelForSubscriptionAsync(
+ Subscription sub) {
+ Map<String, String> subProps = sub.getSubscriptionProperties();
+ try {
+ if (subProps == null || !subProps.containsKey("trace")) {
+ PulsarAdmin admin = sub.getTopic().getBrokerService().getPulsar().getAdminClient();
+ return BrokerTracing.readTraceLevelForTopicAsync(admin, sub.getTopic());
}
- return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+ return CompletableFuture.completedFuture(
+ TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase()));
} catch (IllegalArgumentException e) {
log.warn(
- "Invalid tracing level: {}. Setting to NONE",
- props == null ? "no props" : props.get("trace"));
+ "Invalid tracing level: {}. Setting to NONE for subscription {}",
+ subProps.get("trace"),
+ sub);
+ return CompletableFuture.completedFuture(TraceLevel.NONE);
+ } catch (Throwable t) {
+ log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t);
+ return CompletableFuture.completedFuture(TraceLevel.NONE);
+ }
+ }
+
+ @NotNull
+ private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic) {
+ try {
+ return readTraceLevelForTopicAsync(admin, topic).get();
+ } catch (InterruptedException | ExecutionException e) {
+ log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e);
+ Thread.currentThread().interrupt();
return TraceLevel.NONE;
} catch (Throwable t) {
- log.error("Error getting tracing level", t);
+ log.error("Error getting tracing level for topic {}", topic.getName(), t);
return TraceLevel.NONE;
}
}
+ @NotNull
+ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
+ PulsarAdmin admin, Topic topic) {
+ CompletableFuture<Map<String, String>> propsFuture =
+ admin.topics().getPropertiesAsync(TopicName.get(topic.getName()).getPartitionedTopicName());
+ return propsFuture.handle(
+ (props, ex) -> {
+ if (ex != null) {
+ log.error("Error getting tracing level for topic {}", topic.getName(), ex);
+ return TraceLevel.NONE;
+ }
+
+ try {
+ if (props == null || !props.containsKey("trace")) {
+ return TraceLevel.NONE;
+ }
+
+ return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ log.warn(
+ "Invalid tracing level for topic {}: {}. Setting to NONE",
+ topic.getName(),
+ props.get("trace"));
+ return TraceLevel.NONE;
+ }
+ });
+ }
+
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
From b1df8d8dea18e3fb1c40e036df2c2b45bbb6b4ef Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 16:04:38 -0700
Subject: [PATCH 20/29] part of the feedback addressed, only one level of
onfo(still too verbose), better grabularaty for logger
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 207 ++++----
.../oss/pulsar/jms/tracing/TracingUtils.java | 497 ++++++++----------
.../pulsar/jms/tracing/TracingUtilsTest.java | 6 +-
3 files changed, 304 insertions(+), 406 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 8a77abee..68e34beb 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,6 +15,7 @@
*/
package com.datastax.oss.pulsar.jms.tracing;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
@@ -34,6 +35,7 @@
import com.google.common.util.concurrent.SettableFuture;
import io.netty.buffer.ByteBuf;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
@@ -67,15 +69,7 @@
@Slf4j
public class BrokerTracing implements BrokerInterceptor {
- public enum EventReasons {
- ADMINISTRATIVE,
- COMMANDS,
- MESSAGE,
- TRANSACTION,
- SERVLET,
- }
-
- private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+ private static final TraceLevel defaultTraceLevel = TraceLevel.OFF;
private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
private TraceLevel traceLevel = defaultTraceLevel;
@@ -83,23 +77,26 @@ public enum EventReasons {
private int cacheTraceLevelsDurationSec = 10;
private boolean traceSystemTopics = false;
private boolean traceSchema = false;
- private boolean reduceLevelForNestedComponents = true;
- private static Set<EventReasons> loadEnabledEvents(
+ private static void loadEnabledEvents(
PulsarService pulsarService, Set<EventReasons> enabledEvents) {
- String events =
- pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
- log.debug("read jmsTracingEventList: {}", events);
-
- for (String event : events.split(",")) {
- try {
- enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
- } catch (IllegalArgumentException e) {
- log.error("Invalid event: {}. Skipping", event);
+ Properties props = pulsarService.getConfiguration().getProperties();
+ if (props.contains("jmsTracingEventList")) {
+ String events = props.getProperty("jmsTracingEventList", "");
+ log.debug("read jmsTracingEventList: {}", events);
+
+ enabledEvents.clear();
+ for (String event : events.split(",")) {
+ try {
+ enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
+ } catch (IllegalArgumentException e) {
+ log.error("Invalid event: {}. Skipping", event);
+ }
}
+ } else {
+ log.warn("jmsTracingEventList not set. Using all available.");
+ enabledEvents.addAll(Arrays.asList(EventReasons.values()));
}
-
- return enabledEvents;
}
@NotNull
@@ -163,7 +160,7 @@ public TraceLevel load(Producer producer) {
return BrokerTracing.readTraceLevelForTopic(admin, topic);
} catch (Throwable t) {
log.error("Error getting tracing level", t);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
}
@@ -196,10 +193,10 @@ private static TraceLevel readTraceLevelForSubscription(Subscription sub) {
} catch (InterruptedException | ExecutionException e) {
log.error("Interrupted while getting subscription tracing level for {}", sub, e);
Thread.currentThread().interrupt();
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
} catch (Throwable t) {
log.error("Error getting subscription tracing level for {}", sub, t);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
}
@@ -220,10 +217,10 @@ private static CompletableFuture<TraceLevel> readTraceLevelForSubscriptionAsync(
"Invalid tracing level: {}. Setting to NONE for subscription {}",
subProps.get("trace"),
sub);
- return CompletableFuture.completedFuture(TraceLevel.NONE);
+ return CompletableFuture.completedFuture(TraceLevel.OFF);
} catch (Throwable t) {
log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t);
- return CompletableFuture.completedFuture(TraceLevel.NONE);
+ return CompletableFuture.completedFuture(TraceLevel.OFF);
}
}
@@ -234,10 +231,10 @@ private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic)
} catch (InterruptedException | ExecutionException e) {
log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e);
Thread.currentThread().interrupt();
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
} catch (Throwable t) {
log.error("Error getting tracing level for topic {}", topic.getName(), t);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
}
@@ -250,12 +247,12 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
(props, ex) -> {
if (ex != null) {
log.error("Error getting tracing level for topic {}", topic.getName(), ex);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
try {
if (props == null || !props.containsKey("trace")) {
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
@@ -264,7 +261,7 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
"Invalid tracing level for topic {}: {}. Setting to NONE",
topic.getName(),
props.get("trace"));
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
});
}
@@ -285,10 +282,6 @@ public void initialize(PulsarService pulsarService) {
if (props.containsKey("jmsTracingTraceSchema")) {
traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
}
- if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) {
- reduceLevelForNestedComponents =
- Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
- }
if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
cacheTraceLevelsDurationSec =
Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec"));
@@ -306,44 +299,37 @@ public void close() {
}
private TraceLevel getTracingLevel(Consumer consumer) {
- if (consumer == null) return TraceLevel.NONE;
+ if (consumer == null) return TraceLevel.OFF;
return getTracingLevel(consumer.getSubscription());
}
private TraceLevel getTracingLevel(Subscription sub) {
- if (sub == null) return TraceLevel.NONE;
+ if (sub == null) return TraceLevel.OFF;
- if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE;
+ if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.OFF;
try {
return traceLevelForSubscription.get(sub);
} catch (ExecutionException e) {
log.error("Error getting tracing level", e);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
}
private TraceLevel getTracingLevel(Producer producer) {
- if (producer == null) return TraceLevel.NONE;
+ if (producer == null) return TraceLevel.OFF;
- if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE;
+ if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.OFF;
try {
return traceLevelForProducer.get(producer);
} catch (ExecutionException e) {
log.error("Error getting tracing level", e);
- return TraceLevel.NONE;
+ return TraceLevel.OFF;
}
}
- private TraceLevel getTraceLevelForComponent(TraceLevel current) {
- if (current == TraceLevel.NONE) return TraceLevel.NONE;
- if (reduceLevelForNestedComponents) return TraceLevel.BASIC;
-
- return current;
- }
-
/* ***************************
** Administrative events
******************************/
@@ -351,102 +337,98 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) {
public void onConnectionCreated(ServerCnx cnx) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
- trace("Connection created", traceDetails);
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
}
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
- traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
- trace("Producer created", traceDetails);
+ trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
}
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
- traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
- trace("Producer closed", traceDetails);
+ trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
}
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
- traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
- traceDetails.put(
- "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("consumer", getConsumerDetails(consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
- trace("Consumer created", traceDetails);
+ trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
}
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
- traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
- traceDetails.put(
- "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("consumer", getConsumerDetails(consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
- trace("Consumer closed", traceDetails);
+ trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
}
public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
if (command.hasType()) {
traceDetails.put("type", command.getType().name());
- if (traceLevel != TraceLevel.MINIMAL) {
- traceDetails.put("command", getCommandDetails(traceLevel, command));
- }
+ traceDetails.put("command", getCommandDetails(command));
} else {
traceDetails.put("type", "unknown/null");
}
- trace("Pulsar command called", traceDetails);
+ trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails);
}
public void onConnectionClosed(ServerCnx cnx) {
if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
- trace("Connection closed", traceDetails);
+ trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
}
/* ***************************
@@ -462,16 +444,15 @@ public void beforeSendMessage(
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(subscription);
- if (level == TraceLevel.NONE) return;
+ if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put(
- "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription));
- traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
- traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength));
- traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
+ traceDetails.put("subscription", getSubscriptionDetails(subscription));
+ traceDetails.put("consumer", getConsumerDetails(consumer));
+ traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+ traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
- trace("Before sending message", traceDetails);
+ trace(EventReasons.MESSAGE, "Before sending message", traceDetails);
}
public void onMessagePublish(
@@ -480,15 +461,14 @@ public void onMessagePublish(
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(producer);
- if (level == TraceLevel.NONE) return;
+ if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put(
- "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
+ traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
- trace("Message publish", traceDetails);
+ trace(EventReasons.MESSAGE, "Message publish", traceDetails);
}
public void messageProduced(
@@ -501,16 +481,15 @@ public void messageProduced(
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(producer);
- if (level == TraceLevel.NONE) return;
+ if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put(
- "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
- trace("Message produced", traceDetails);
+ trace(EventReasons.MESSAGE, "Message produced", traceDetails);
}
public void messageDispatched(
@@ -518,32 +497,28 @@ public void messageDispatched(
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(consumer);
- if (level == TraceLevel.NONE) return;
+ if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
- traceDetails.put(
- "subscription",
- getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("consumer", getConsumerDetails(consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
- trace("After dispatching message", traceDetails);
+ trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(consumer);
- if (level == TraceLevel.NONE) return;
+ if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
- traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
- traceDetails.put(
- "subscription",
- getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("consumer", getConsumerDetails(consumer));
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
Map<String, Object> ackDetails = new TreeMap<>();
if (ackCmd.hasAckType()) {
@@ -575,7 +550,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
traceDetails.put("ack", ackDetails);
- trace("Message acked", traceDetails);
+ trace(EventReasons.MESSAGE, "Message acked", traceDetails);
}
@NotNull
@@ -595,24 +570,24 @@ private static String formatMessageId(MessageIdData x) {
public void txnOpened(long tcId, String txnID) {
if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("tcId", tcId);
traceDetails.put("txnID", txnID);
- trace("Transaction opened", traceDetails);
+ trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails);
}
public void txnEnded(String txnID, long txnAction) {
if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
- if (traceLevel == TraceLevel.NONE) return;
+ if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("txnID", txnID);
traceDetails.put("txnAction", txnAction);
- trace("Transaction closed", traceDetails);
+ trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails);
}
/* ***************************
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 4e6311df..d995387a 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -23,6 +23,7 @@
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -44,15 +45,38 @@
@Slf4j
public class TracingUtils {
- private static final org.slf4j.Logger traceLogger =
- org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+ public enum EventReasons {
+ ADMINISTRATIVE,
+ COMMANDS,
+ MESSAGE,
+ TRANSACTION,
+ SERVLET,
+ }
@FunctionalInterface
public interface Tracer {
- void trace(String message);
+ void trace(EventReasons reason, String message);
}
- public static final Tracer SLF4J_TRACER = traceLogger::info;
+ public static class Slf4jTracer implements Tracer {
+ private static final Map<EventReasons, org.slf4j.Logger> traceLoggers = new HashMap<>();
+
+ static {
+ for (EventReasons reason : EventReasons.values()) {
+ traceLoggers.put(
+ reason,
+ org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase()));
+ }
+ }
+
+ @Override
+ public void trace(EventReasons reason, String message) {
+ traceLoggers.get(reason).info(message);
+ }
+ }
+
+ public static final Tracer SLF4J_TRACER = new Slf4jTracer();
private static final ObjectMapper mapper =
new ObjectMapper()
@@ -60,24 +84,23 @@ public interface Tracer {
.enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
public enum TraceLevel {
- NONE,
- MINIMAL,
- BASIC,
- FULL
+ OFF,
+ ON
}
- public static void trace(String message, Map<String, Object> traceDetails) {
- trace(SLF4J_TRACER, message, traceDetails);
+ public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
+ trace(SLF4J_TRACER, reason, message, traceDetails);
}
- public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
+ public static void trace(
+ Tracer tracer, EventReasons reason, String message, Map<String, Object> traceDetails) {
Map<String, Object> trace = new TreeMap<>();
trace.put("eventType", message);
trace.put("traceDetails", traceDetails);
try {
String loggableJsonString = mapper.writeValueAsString(trace);
- tracer.trace(loggableJsonString);
+ tracer.trace(reason, loggableJsonString);
} catch (JsonProcessingException e) {
log.error(
"Failed to serialize trace event type '{}' as json, traceDetails: {}",
@@ -87,24 +110,21 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
}
}
- public static Map<String, Object> getCommandDetails(TraceLevel level, BaseCommand command) {
+ public static Map<String, Object> getCommandDetails(BaseCommand command) {
if (command == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateCommandDetails(level, command, details);
+ populateCommandDetails(command, details);
return details;
}
private static void populateCommandDetails(
- TraceLevel level, BaseCommand command, Map<String, Object> traceDetails) {
+ BaseCommand command, Map<String, Object> traceDetails) {
if (command == null) {
return;
}
- if (level == TraceLevel.NONE) {
- return;
- }
if (!command.hasType()) {
return;
@@ -113,178 +133,178 @@ private static void populateCommandDetails(
// trace all params otherwise
switch (command.getType()) {
case CONNECT:
- populateByReflection(level, command.getConnect(), traceDetails);
+ populateByReflection(command.getConnect(), traceDetails);
break;
case CONNECTED:
- populateByReflection(level, command.getConnected(), traceDetails);
+ populateByReflection(command.getConnected(), traceDetails);
break;
case SUBSCRIBE:
- populateByReflection(level, command.getSubscribe(), traceDetails);
+ populateByReflection(command.getSubscribe(), traceDetails);
break;
case PRODUCER:
- populateByReflection(level, command.getProducer(), traceDetails);
+ populateByReflection(command.getProducer(), traceDetails);
break;
case SEND:
- populateByReflection(level, command.getSend(), traceDetails);
+ populateByReflection(command.getSend(), traceDetails);
break;
case SEND_RECEIPT:
- populateByReflection(level, command.getSendReceipt(), traceDetails);
+ populateByReflection(command.getSendReceipt(), traceDetails);
break;
case SEND_ERROR:
- populateByReflection(level, command.getSendError(), traceDetails);
+ populateByReflection(command.getSendError(), traceDetails);
break;
case MESSAGE:
- populateByReflection(level, command.getMessage(), traceDetails);
+ populateByReflection(command.getMessage(), traceDetails);
break;
case ACK:
- populateByReflection(level, command.getAck(), traceDetails);
+ populateByReflection(command.getAck(), traceDetails);
break;
case FLOW:
- populateByReflection(level, command.getFlow(), traceDetails);
+ populateByReflection(command.getFlow(), traceDetails);
break;
case UNSUBSCRIBE:
- populateByReflection(level, command.getUnsubscribe(), traceDetails);
+ populateByReflection(command.getUnsubscribe(), traceDetails);
break;
case SUCCESS:
- populateByReflection(level, command.getSuccess(), traceDetails);
+ populateByReflection(command.getSuccess(), traceDetails);
break;
case ERROR:
- populateByReflection(level, command.getError(), traceDetails);
+ populateByReflection(command.getError(), traceDetails);
break;
case CLOSE_PRODUCER:
- populateByReflection(level, command.getCloseProducer(), traceDetails);
+ populateByReflection(command.getCloseProducer(), traceDetails);
break;
case CLOSE_CONSUMER:
- populateByReflection(level, command.getCloseConsumer(), traceDetails);
+ populateByReflection(command.getCloseConsumer(), traceDetails);
break;
case PRODUCER_SUCCESS:
- populateByReflection(level, command.getProducerSuccess(), traceDetails);
+ populateByReflection(command.getProducerSuccess(), traceDetails);
break;
case PING:
- populateByReflection(level, command.getPing(), traceDetails);
+ populateByReflection(command.getPing(), traceDetails);
break;
case PONG:
- populateByReflection(level, command.getPong(), traceDetails);
+ populateByReflection(command.getPong(), traceDetails);
break;
case REDELIVER_UNACKNOWLEDGED_MESSAGES:
- populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails);
+ populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
break;
case PARTITIONED_METADATA:
- populateByReflection(level, command.getPartitionMetadata(), traceDetails);
+ populateByReflection(command.getPartitionMetadata(), traceDetails);
break;
case PARTITIONED_METADATA_RESPONSE:
- populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails);
+ populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
break;
case LOOKUP:
- populateByReflection(level, command.getLookupTopic(), traceDetails);
+ populateByReflection(command.getLookupTopic(), traceDetails);
break;
case LOOKUP_RESPONSE:
- populateByReflection(level, command.getLookupTopicResponse(), traceDetails);
+ populateByReflection(command.getLookupTopicResponse(), traceDetails);
break;
case CONSUMER_STATS:
- populateByReflection(level, command.getConsumerStats(), traceDetails);
+ populateByReflection(command.getConsumerStats(), traceDetails);
break;
case CONSUMER_STATS_RESPONSE:
- populateByReflection(level, command.getConsumerStatsResponse(), traceDetails);
+ populateByReflection(command.getConsumerStatsResponse(), traceDetails);
break;
case REACHED_END_OF_TOPIC:
- populateByReflection(level, command.getReachedEndOfTopic(), traceDetails);
+ populateByReflection(command.getReachedEndOfTopic(), traceDetails);
break;
case SEEK:
- populateByReflection(level, command.getSeek(), traceDetails);
+ populateByReflection(command.getSeek(), traceDetails);
break;
case GET_LAST_MESSAGE_ID:
- populateByReflection(level, command.getGetLastMessageId(), traceDetails);
+ populateByReflection(command.getGetLastMessageId(), traceDetails);
break;
case GET_LAST_MESSAGE_ID_RESPONSE:
- populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails);
+ populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
break;
case ACTIVE_CONSUMER_CHANGE:
- populateByReflection(level, command.getActiveConsumerChange(), traceDetails);
+ populateByReflection(command.getActiveConsumerChange(), traceDetails);
break;
case GET_TOPICS_OF_NAMESPACE:
- populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails);
+ populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
break;
case GET_TOPICS_OF_NAMESPACE_RESPONSE:
- populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails);
+ populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
break;
case GET_SCHEMA:
- populateByReflection(level, command.getGetSchema(), traceDetails);
+ populateByReflection(command.getGetSchema(), traceDetails);
break;
case GET_SCHEMA_RESPONSE:
- populateByReflection(level, command.getGetSchemaResponse(), traceDetails);
+ populateByReflection(command.getGetSchemaResponse(), traceDetails);
break;
case AUTH_CHALLENGE:
- populateByReflection(level, command.getAuthChallenge(), traceDetails);
+ populateByReflection(command.getAuthChallenge(), traceDetails);
break;
case AUTH_RESPONSE:
- populateByReflection(level, command.getAuthResponse(), traceDetails);
+ populateByReflection(command.getAuthResponse(), traceDetails);
break;
case ACK_RESPONSE:
- populateByReflection(level, command.getAckResponse(), traceDetails);
+ populateByReflection(command.getAckResponse(), traceDetails);
break;
case GET_OR_CREATE_SCHEMA:
- populateByReflection(level, command.getGetOrCreateSchema(), traceDetails);
+ populateByReflection(command.getGetOrCreateSchema(), traceDetails);
break;
case GET_OR_CREATE_SCHEMA_RESPONSE:
- populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails);
+ populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
break;
case NEW_TXN:
- populateByReflection(level, command.getNewTxn(), traceDetails);
+ populateByReflection(command.getNewTxn(), traceDetails);
break;
case NEW_TXN_RESPONSE:
- populateByReflection(level, command.getNewTxnResponse(), traceDetails);
+ populateByReflection(command.getNewTxnResponse(), traceDetails);
break;
case ADD_PARTITION_TO_TXN:
- populateByReflection(level, command.getAddPartitionToTxn(), traceDetails);
+ populateByReflection(command.getAddPartitionToTxn(), traceDetails);
break;
case ADD_PARTITION_TO_TXN_RESPONSE:
- populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails);
+ populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
break;
case ADD_SUBSCRIPTION_TO_TXN:
- populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails);
+ populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
break;
case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
- populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails);
+ populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
break;
case END_TXN:
- populateByReflection(level, command.getEndTxn(), traceDetails);
+ populateByReflection(command.getEndTxn(), traceDetails);
break;
case END_TXN_RESPONSE:
- populateByReflection(level, command.getEndTxnResponse(), traceDetails);
+ populateByReflection(command.getEndTxnResponse(), traceDetails);
break;
case END_TXN_ON_PARTITION:
- populateByReflection(level, command.getEndTxnOnPartition(), traceDetails);
+ populateByReflection(command.getEndTxnOnPartition(), traceDetails);
break;
case END_TXN_ON_PARTITION_RESPONSE:
- populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails);
+ populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
break;
case END_TXN_ON_SUBSCRIPTION:
- populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails);
+ populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
break;
case END_TXN_ON_SUBSCRIPTION_RESPONSE:
- populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails);
+ populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
break;
case TC_CLIENT_CONNECT_REQUEST:
- populateByReflection(level, command.getTcClientConnectRequest(), traceDetails);
+ populateByReflection(command.getTcClientConnectRequest(), traceDetails);
break;
case TC_CLIENT_CONNECT_RESPONSE:
- populateByReflection(level, command.getTcClientConnectResponse(), traceDetails);
+ populateByReflection(command.getTcClientConnectResponse(), traceDetails);
break;
case WATCH_TOPIC_LIST:
- populateByReflection(level, command.getWatchTopicList(), traceDetails);
+ populateByReflection(command.getWatchTopicList(), traceDetails);
break;
case WATCH_TOPIC_LIST_SUCCESS:
- populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails);
+ populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
break;
case WATCH_TOPIC_UPDATE:
- populateByReflection(level, command.getWatchTopicUpdate(), traceDetails);
+ populateByReflection(command.getWatchTopicUpdate(), traceDetails);
break;
case WATCH_TOPIC_LIST_CLOSE:
- populateByReflection(level, command.getWatchTopicListClose(), traceDetails);
+ populateByReflection(command.getWatchTopicListClose(), traceDetails);
break;
case TOPIC_MIGRATED:
- populateByReflection(level, command.getTopicMigrated(), traceDetails);
+ populateByReflection(command.getTopicMigrated(), traceDetails);
break;
default:
log.error("Unknown command type: {}", command.getType());
@@ -292,7 +312,7 @@ private static void populateCommandDetails(
}
}
- private static final Set<String> fullTraceFields =
+ private static final Set<String> skipTraceFields =
Sets.newHashSet(
"authdata",
"authmethod",
@@ -302,8 +322,7 @@ private static void populateCommandDetails(
"originalprincipal",
"schema");
- private static void populateByReflection(
- TraceLevel level, Object command, Map<String, Object> traceDetails) {
+ private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
if (command == null) {
return;
}
@@ -320,7 +339,7 @@ private static void populateByReflection(
return false;
}
String fieldName = method.getName().substring(3);
- return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase());
+ return !skipTraceFields.contains(fieldName.toLowerCase());
})
.filter(
method -> {
@@ -356,12 +375,12 @@ private static void populateByReflection(
if (value instanceof byte[]
|| value instanceof ByteBuf
|| value instanceof ByteBuffer) {
- int size = 0;
+ final int size;
if (value instanceof byte[]) {
size = ((byte[]) value).length;
} else if (value instanceof ByteBuf) {
size = ((ByteBuf) value).readableBytes();
- } else if (value instanceof ByteBuffer) {
+ } else {
size = ((ByteBuffer) value).remaining();
}
traceDetails.put(fieldName + "_size", size);
@@ -373,7 +392,7 @@ private static void populateByReflection(
.getCanonicalName()
.contains("org.apache.pulsar.common.api.proto")) {
Map<String, Object> details = new TreeMap<>();
- populateByReflection(level, value, details);
+ populateByReflection(value, details);
traceDetails.put(fieldName, details);
} else {
traceDetails.put(fieldName, value);
@@ -384,53 +403,35 @@ private static void populateByReflection(
});
}
- public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+ public static Map<String, Object> getConnectionDetails(ServerCnx cnx) {
if (cnx == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateConnectionDetails(level, cnx, details);
+ populateConnectionDetails(cnx, details);
return details;
}
- private static void populateConnectionDetails(
- TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object> traceDetails) {
if (cnx == null) {
return;
}
- switch (level) {
- case MINIMAL:
- traceDetails.put("clientAddress", cnx.clientAddress().toString());
- traceDetails.put("authRole", cnx.getAuthRole());
- break;
- case BASIC:
- populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
-
- traceDetails.put("clientVersion", cnx.getClientVersion());
- traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
- break;
- case FULL:
- populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
-
- traceDetails.put("authMethod", cnx.getAuthMethod());
- traceDetails.put(
- "authMethodName",
- cnx.getAuthenticationProvider() == null
- ? "no provider"
- : cnx.getAuthenticationProvider().getAuthMethodName());
-
- AuthenticationDataSource authData = cnx.getAuthenticationData();
- if (authData != null) {
- traceDetails.put("authData", getAuthDataDetails(authData));
- }
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
+ traceDetails.put("clientAddress", cnx.clientAddress());
+ traceDetails.put("authRole", cnx.getAuthRole());
+ traceDetails.put("clientVersion", cnx.getClientVersion());
+ traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+ traceDetails.put("authMethod", cnx.getAuthMethod());
+ traceDetails.put(
+ "authMethodName",
+ cnx.getAuthenticationProvider() == null
+ ? "no provider"
+ : cnx.getAuthenticationProvider().getAuthMethodName());
+
+ AuthenticationDataSource authData = cnx.getAuthenticationData();
+ if (authData != null) {
+ traceDetails.put("authData", getAuthDataDetails(authData));
}
}
@@ -463,260 +464,178 @@ private static void populateAuthDataDetails(
}
}
- public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+ public static Map<String, Object> getSubscriptionDetails(Subscription sub) {
if (sub == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateSubscriptionDetails(level, sub, details);
+ populateSubscriptionDetails(sub, details);
return details;
}
private static void populateSubscriptionDetails(
- TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+ Subscription sub, Map<String, Object> traceDetails) {
if (sub == null) {
return;
}
- switch (level) {
- case MINIMAL:
- traceDetails.put("name", sub.getName());
- traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- traceDetails.put("type", sub.getType().name());
- break;
- case BASIC:
- populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
-
- if (sub.getConsumers() != null) {
- traceDetails.put("numberOfConsumers", sub.getConsumers().size());
- traceDetails.put(
- "namesOfConsumers",
- sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
- }
+ traceDetails.put("name", sub.getName());
+ traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("type", sub.getType().name());
- break;
- case FULL:
- populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
-
- traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
+ if (sub.getConsumers() != null) {
+ traceDetails.put("numberOfConsumers", sub.getConsumers().size());
+ traceDetails.put(
+ "namesOfConsumers",
+ sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
}
+
+ traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
}
- public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+ public static Map<String, Object> getConsumerDetails(Consumer consumer) {
if (consumer == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateConsumerDetails(level, consumer, details);
+ populateConsumerDetails(consumer, details);
return details;
}
- private static void populateConsumerDetails(
- TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+ private static void populateConsumerDetails(Consumer consumer, Map<String, Object> traceDetails) {
if (consumer == null) {
return;
}
- switch (level) {
- case MINIMAL:
- traceDetails.put("name", consumer.consumerName());
- traceDetails.put("consumerId", consumer.consumerId());
- Subscription sub = consumer.getSubscription();
- if (sub != null) {
- traceDetails.put("subscriptionName", sub.getName());
- traceDetails.put(
- "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- }
- break;
- case BASIC:
- populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+ traceDetails.put("name", consumer.consumerName());
+ traceDetails.put("consumerId", consumer.consumerId());
+ Subscription sub = consumer.getSubscription();
+ if (sub != null) {
+ traceDetails.put("subscriptionName", sub.getName());
+ traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ }
- traceDetails.put("priorityLevel", consumer.getPriorityLevel());
- traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
- traceDetails.put("clientAddress", consumer.getClientAddress());
- break;
- case FULL:
- populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+ traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+ traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
+ traceDetails.put("clientAddress", consumer.getClientAddress());
- traceDetails.put("metadata", consumer.getMetadata());
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
+ traceDetails.put("metadata", consumer.getMetadata());
}
- public static Map<String, Object> getProducerDetails(
- TraceLevel level, Producer producer, boolean traceSchema) {
+ public static Map<String, Object> getProducerDetails(Producer producer, boolean traceSchema) {
if (producer == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateProducerDetails(level, producer, details, traceSchema);
+ populateProducerDetails(producer, details, traceSchema);
return details;
}
private static void populateProducerDetails(
- TraceLevel level, Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
+ Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
if (producer == null) {
return;
}
- switch (level) {
- case MINIMAL:
- traceDetails.put("producerId", producer.getProducerId());
- traceDetails.put("producerName", producer.getProducerName());
- traceDetails.put(
- "accessMode",
- producer.getAccessMode() == null ? null : producer.getAccessMode().name());
- if (producer.getTopic() != null) {
- traceDetails.put(
- "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
- }
- break;
- case BASIC:
- populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema);
+ traceDetails.put("producerId", producer.getProducerId());
+ traceDetails.put("producerName", producer.getProducerName());
+ traceDetails.put(
+ "accessMode", producer.getAccessMode() == null ? null : producer.getAccessMode().name());
+ if (producer.getTopic() != null) {
+ traceDetails.put(
+ "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+ }
- traceDetails.put("clientAddress", producer.getClientAddress());
- break;
- case FULL:
- populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema);
+ traceDetails.put("clientAddress", producer.getClientAddress());
- traceDetails.put("metadata", producer.getMetadata());
+ traceDetails.put("metadata", producer.getMetadata());
- if (traceSchema && producer.getSchemaVersion() != null) {
- final String schemaVersion;
- if (producer.getSchemaVersion() == SchemaVersion.Empty) {
- schemaVersion = "Empty";
- } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
- schemaVersion = "Latest";
- } else {
- schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
- }
- traceDetails.put("schemaVersion", schemaVersion);
- }
- traceDetails.put("remoteCluster", producer.getRemoteCluster());
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
+ if (traceSchema && producer.getSchemaVersion() != null) {
+ final String schemaVersion;
+ if (producer.getSchemaVersion() == SchemaVersion.Empty) {
+ schemaVersion = "Empty";
+ } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
+ schemaVersion = "Latest";
+ } else {
+ schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
+ }
+ traceDetails.put("schemaVersion", schemaVersion);
}
+ traceDetails.put("remoteCluster", producer.getRemoteCluster());
}
- public static Map<String, Object> getMessageMetadataDetails(
- TraceLevel level, MessageMetadata msgMetadata) {
+ public static Map<String, Object> getMessageMetadataDetails(MessageMetadata msgMetadata) {
if (msgMetadata == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateMessageMetadataDetails(level, msgMetadata, details);
+ populateMessageMetadataDetails(msgMetadata, details);
return details;
}
private static void populateMessageMetadataDetails(
- TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+ MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
if (msgMetadata == null) {
return;
}
- switch (level) {
- case MINIMAL:
- if (msgMetadata.hasPartitionKey()) {
- traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
- }
- if (msgMetadata.hasSequenceId()) {
- traceDetails.put("sequenceId", msgMetadata.getSequenceId());
- }
- if (msgMetadata.hasProducerName()) {
- traceDetails.put("producerName", msgMetadata.getProducerName());
- }
- break;
- case BASIC:
- populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
-
- if (msgMetadata.hasUncompressedSize()) {
- traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
- }
- if (msgMetadata.hasNumMessagesInBatch()) {
- traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
- }
- traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
- break;
- case FULL:
- populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
-
- if (msgMetadata.hasPublishTime()) {
- traceDetails.put("publishTime", msgMetadata.getPublishTime());
- }
- if (msgMetadata.hasEventTime()) {
- traceDetails.put("eventTime", msgMetadata.getEventTime());
- }
- if (msgMetadata.hasReplicatedFrom()) {
- traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
- }
- if (msgMetadata.hasUuid()) {
- traceDetails.put("uuid", msgMetadata.getUuid());
- }
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
+ if (msgMetadata.hasPartitionKey()) {
+ traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+ }
+ if (msgMetadata.hasSequenceId()) {
+ traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+ }
+ if (msgMetadata.hasProducerName()) {
+ traceDetails.put("producerName", msgMetadata.getProducerName());
+ }
+
+ if (msgMetadata.hasUncompressedSize()) {
+ traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+ }
+ if (msgMetadata.hasNumMessagesInBatch()) {
+ traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+ }
+ traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+
+ if (msgMetadata.hasPublishTime()) {
+ traceDetails.put("publishTime", msgMetadata.getPublishTime());
+ }
+ if (msgMetadata.hasEventTime()) {
+ traceDetails.put("eventTime", msgMetadata.getEventTime());
+ }
+ if (msgMetadata.hasReplicatedFrom()) {
+ traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+ }
+ if (msgMetadata.hasUuid()) {
+ traceDetails.put("uuid", msgMetadata.getUuid());
}
}
- public static Map<String, Object> getEntryDetails(
- TraceLevel level, Entry entry, int maxBinaryDataLength) {
+ public static Map<String, Object> getEntryDetails(Entry entry, int maxBinaryDataLength) {
if (entry == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateEntryDetails(level, entry, details, maxBinaryDataLength);
+ populateEntryDetails(entry, details, maxBinaryDataLength);
return details;
}
private static void populateEntryDetails(
- TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+ Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
if (entry == null) {
return;
}
- switch (level) {
- case MINIMAL:
- traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
- break;
- case BASIC:
- populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength);
+ traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
- traceDetails.put("length", entry.getLength());
- break;
- case FULL:
- populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength);
+ traceDetails.put("length", entry.getLength());
- traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
- break;
- case NONE:
- break;
- default:
- log.warn("Unknown tracing level: {}", level);
- break;
- }
+ traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
}
public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 220fee87..fbae35b2 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -32,11 +32,15 @@ class TracingUtilsTest {
private Tracer mockTracer =
new Tracer() {
@Override
- public void trace(String msg) {
+ public void trace(EventReasons reason, String msg) {
traces.add(msg);
}
};
+ private static void trace(Tracer mockTracer, String msg, Map<String, Object> traceDetails) {
+ TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails);
+ }
+
@Test
void traceTest() {
traces.clear();
From 8e8c866c6615c4b5a39ef92d594b630e74f1af26 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 16:51:21 -0700
Subject: [PATCH 21/29] resolve hostname of the client
---
.../oss/pulsar/jms/tracing/TracingUtils.java | 50 +++++++++++++++++--
1 file changed, 47 insertions(+), 3 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index d995387a..5caa7e36 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -18,9 +18,14 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
@@ -88,6 +93,44 @@ public enum TraceLevel {
ON
}
+ private static final LoadingCache<String, String> ipResolverCache =
+ CacheBuilder.newBuilder()
+ .maximumSize(10_000L)
+ .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .build(
+ new CacheLoader<String, String>() {
+ public String load(String clientAddress) {
+ // Dn resolution can be slow in some cases
+ // and we do not want to create too many requests to DNS,
+ // so we cache the result
+ log.info("resolving DNS for {}", clientAddress);
+ try {
+ InetAddress address = InetAddress.getByName(clientAddress);
+ String hostName = address.getCanonicalHostName();
+ if (log.isDebugEnabled()) {
+ log.debug("Resolved DNS for {} to {}", clientAddress, hostName);
+ }
+ return hostName;
+ } catch (UnknownHostException e) {
+ log.error("Failed to resolve DNS for {}", clientAddress, e);
+ return clientAddress;
+ }
+ }
+ });
+
+ public static String hostNameOf(String clientAddress) {
+ if (clientAddress == null || clientAddress.isEmpty()) {
+ return "unknown/null";
+ }
+
+ try {
+ return ipResolverCache.get(clientAddress);
+ } catch (Throwable t) {
+ log.error("Failed to resolve DNS for {}", clientAddress, t);
+ return clientAddress;
+ }
+ }
+
public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
trace(SLF4J_TRACER, reason, message, traceDetails);
}
@@ -418,7 +461,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
return;
}
- traceDetails.put("clientAddress", cnx.clientAddress());
+ traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress()));
+ traceDetails.put("clientSocket", cnx.clientAddress());
traceDetails.put("authRole", cnx.getAuthRole());
traceDetails.put("clientVersion", cnx.getClientVersion());
traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
@@ -519,7 +563,7 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
traceDetails.put("priorityLevel", consumer.getPriorityLevel());
traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
- traceDetails.put("clientAddress", consumer.getClientAddress());
+ traceDetails.put("clientAddress", hostNameOf(consumer.getClientAddress()));
traceDetails.put("metadata", consumer.getMetadata());
}
@@ -549,7 +593,7 @@ private static void populateProducerDetails(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
}
- traceDetails.put("clientAddress", producer.getClientAddress());
+ traceDetails.put("clientAddress", hostNameOf(producer.getClientAddress()));
traceDetails.put("metadata", producer.getMetadata());
From f16ac60b5e0d2c012d5f7129884126dea7c18c57 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 17:55:09 -0700
Subject: [PATCH 22/29] parse payload out of the ByteBuf
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 6 ++-
.../oss/pulsar/jms/tracing/TracingUtils.java | 47 +++++++++++++++++--
2 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 68e34beb..bdcf22ac 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -466,7 +466,11 @@ public void onMessagePublish(
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
- traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
+
+ Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+ traceByteBuf(
+ "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+ traceDetails.put("headersAndPayload", headersAndPayloadDetails);
trace(EventReasons.MESSAGE, "Message publish", traceDetails);
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 5caa7e36..12af1721 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -27,6 +27,7 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -45,7 +46,10 @@
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.compression.CompressionCodec;
+import org.apache.pulsar.common.compression.CompressionCodecProvider;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
import org.apache.pulsar.common.protocol.schema.SchemaVersion;
@Slf4j
@@ -709,12 +713,45 @@ private static void populatePublishContext(
public static void traceByteBuf(
String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
if (buf == null || maxBinaryDataLength <= 0) return;
+ try {
- if (buf.readableBytes() < maxBinaryDataLength) {
- traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
- } else {
- traceDetails.put(
- key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()));
+ final ByteBuf metadataAndPayload = buf.retainedDuplicate();
+ ByteBuf uncompressedPayload = null;
+ try {
+ // advance readerIndex
+ MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
+
+ // todo: do we need to trace this metadata?
+ populateMessageMetadataDetails(metadata, traceDetails);
+
+ // Decode if needed
+ CompressionCodec codec =
+ CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
+ uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
+
+ // todo: does this require additional steps if messages are batched?
+ if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) {
+ String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8);
+ traceDetails.put(key, dataAsString);
+ } else {
+ String dataAsString =
+ uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8);
+ traceDetails.put(key, dataAsString + "...");
+ }
+ } finally {
+ metadataAndPayload.release();
+ if (uncompressedPayload != null) {
+ uncompressedPayload.release();
+ }
+ }
+ } catch (Throwable t) {
+ log.error("Failed to convert ByteBuf to string", t);
+ if (buf.readableBytes() < maxBinaryDataLength + 3) {
+ traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
+ } else {
+ traceDetails.put(
+ key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "...");
+ }
}
}
}
From 277e0b45f20b9c4503e14b7efb6680d1c603568e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 18:02:03 -0700
Subject: [PATCH 23/29] fix test, some wording
---
.../datastax/oss/pulsar/jms/tracing/BrokerTracing.java | 8 ++++++--
.../com/datastax/oss/pulsar/jms/tracing/TracingUtils.java | 2 +-
.../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java | 4 ++--
3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index bdcf22ac..dfbfdec7 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -470,7 +470,7 @@ public void onMessagePublish(
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
traceByteBuf(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
- traceDetails.put("headersAndPayload", headersAndPayloadDetails);
+ traceDetails.put("payload", headersAndPayloadDetails);
trace(EventReasons.MESSAGE, "Message publish", traceDetails);
}
@@ -508,7 +508,11 @@ public void messageDispatched(
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("messageId", ledgerId + ":" + entryId);
- traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
+
+ Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+ traceByteBuf(
+ "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+ traceDetails.put("payload", headersAndPayloadDetails);
trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 12af1721..64e4aa92 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -683,7 +683,7 @@ private static void populateEntryDetails(
traceDetails.put("length", entry.getLength());
- traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
+ traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
}
public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index fbae35b2..71e01bba 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -118,7 +118,7 @@ void traceByteBufTest() {
traceDetails.clear();
traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
assertEquals(1, traceDetails.size());
- assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length());
- assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+ assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
+ assertTrue(((String) traceDetails.get("key")).startsWith("0x000102"));
}
}
From 87c640897004ed0538d07e0e00a888258fe65762 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 3 May 2024 15:02:07 -0700
Subject: [PATCH 24/29] WIP reduce verbosity
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 47 ++++++++++++++-----
.../oss/pulsar/jms/tracing/TracingUtils.java | 30 ++++++------
2 files changed, 50 insertions(+), 27 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dfbfdec7..0af2d244 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -64,6 +64,9 @@
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.intercept.InterceptException;
import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
+import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
+import org.apache.pulsar.common.util.DateFormatter;
import org.jetbrains.annotations.NotNull;
@Slf4j
@@ -351,7 +354,6 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
@@ -365,10 +367,17 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
+ PublisherStatsImpl stats = producer.getStats();
+ traceDetails.put("connectedSince", stats.getConnectedSince());
+ traceDetails.put("closedAt", DateFormatter.now());
+ traceDetails.put("averageMsgSize", stats.getAverageMsgSize());
+ traceDetails.put("msgRateIn", stats.getMsgRateIn());
+ traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
+ // no message count in stats? stats.getCount() is not it
+
trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
}
@@ -379,7 +388,6 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
@@ -394,11 +402,22 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
if (traceLevel == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
+ ConsumerStatsImpl stats = consumer.getStats();
+
+ traceDetails.put("connectedSince", stats.getConnectedSince());
+ traceDetails.put("closedAt", DateFormatter.now());
+ traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
+ traceDetails.put("msgRateOut", stats.getMsgRateOut());
+ traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut());
+ traceDetails.put("msgOutCounter", stats.getMsgOutCounter());
+ traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
+ traceDetails.put("unackedMessages", stats.getUnackedMessages());
+ traceDetails.put("messageAckRate", stats.getMessageAckRate());
+
trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
}
@@ -452,7 +471,7 @@ public void beforeSendMessage(
traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
- trace(EventReasons.MESSAGE, "Before sending message", traceDetails);
+ trace(EventReasons.MESSAGE, "Message read", traceDetails);
}
public void onMessagePublish(
@@ -472,7 +491,7 @@ public void onMessagePublish(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "Message publish", traceDetails);
+ trace(EventReasons.MESSAGE, "Message received", traceDetails);
}
public void messageProduced(
@@ -493,7 +512,7 @@ public void messageProduced(
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
- trace(EventReasons.MESSAGE, "Message produced", traceDetails);
+ trace(EventReasons.MESSAGE, "Message stored", traceDetails);
}
public void messageDispatched(
@@ -504,17 +523,18 @@ public void messageDispatched(
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("consumer", getConsumerDetails(consumer));
- traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+ if (consumer != null) {
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+ }
traceDetails.put("messageId", ledgerId + ":" + entryId);
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
traceByteBuf(
- "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+ "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
+ trace(EventReasons.MESSAGE, "Message dispatched", traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -524,9 +544,10 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("consumer", getConsumerDetails(consumer));
- traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+ if (consumer != null) {
+ traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+ }
Map<String, Object> ackDetails = new TreeMap<>();
if (ackCmd.hasAckType()) {
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 64e4aa92..25f933e1 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -34,6 +34,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Entry;
@@ -100,7 +101,7 @@ public enum TraceLevel {
private static final LoadingCache<String, String> ipResolverCache =
CacheBuilder.newBuilder()
.maximumSize(10_000L)
- .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+ .expireAfterWrite(4, TimeUnit.HOURS)
.build(
new CacheLoader<String, String>() {
public String load(String clientAddress) {
@@ -464,18 +465,15 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
if (cnx == null) {
return;
}
-
- traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress()));
- traceDetails.put("clientSocket", cnx.clientAddress());
+ traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
traceDetails.put("authRole", cnx.getAuthRole());
+ traceDetails.put("principal", cnx.getPrincipal());
traceDetails.put("clientVersion", cnx.getClientVersion());
traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
traceDetails.put("authMethod", cnx.getAuthMethod());
- traceDetails.put(
- "authMethodName",
- cnx.getAuthenticationProvider() == null
- ? "no provider"
- : cnx.getAuthenticationProvider().getAuthMethodName());
+ if (cnx.getAuthenticationProvider() != null) {
+ traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+ }
AuthenticationDataSource authData = cnx.getAuthenticationData();
if (authData != null) {
@@ -534,10 +532,10 @@ private static void populateSubscriptionDetails(
if (sub.getConsumers() != null) {
traceDetails.put("numberOfConsumers", sub.getConsumers().size());
- traceDetails.put(
- "namesOfConsumers",
- sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
}
+ traceDetails.put("isReplicated", sub.isReplicated());
+ traceDetails.put("numberOfEntriesDelayed", sub.getNumberOfEntriesDelayed());
+ traceDetails.put("numberOfEntriesInBacklog", sub.getNumberOfEntriesInBacklog(false));
traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
}
@@ -567,9 +565,11 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
traceDetails.put("priorityLevel", consumer.getPriorityLevel());
traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
- traceDetails.put("clientAddress", hostNameOf(consumer.getClientAddress()));
+ traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress()));
traceDetails.put("metadata", consumer.getMetadata());
+ traceDetails.put("unackedMessages", consumer.getUnackedMessages());
+ traceDetails.put("authRole", consumer.cnx().getAuthRole());
}
public static Map<String, Object> getProducerDetails(Producer producer, boolean traceSchema) {
@@ -597,7 +597,7 @@ private static void populateProducerDetails(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
}
- traceDetails.put("clientAddress", hostNameOf(producer.getClientAddress()));
+ traceDetails.put("clientHost", hostNameOf(producer.getClientAddress()));
traceDetails.put("metadata", producer.getMetadata());
@@ -613,6 +613,8 @@ private static void populateProducerDetails(
traceDetails.put("schemaVersion", schemaVersion);
}
traceDetails.put("remoteCluster", producer.getRemoteCluster());
+
+ traceDetails.put("authRole", producer.getCnx().getAuthRole());
}
public static Map<String, Object> getMessageMetadataDetails(MessageMetadata msgMetadata) {
From 398fa3f490e4988cef4cf28d7121667bec0a7f21 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 6 May 2024 16:28:49 -0700
Subject: [PATCH 25/29] reduced traces verbosity, added some details
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 108 +++++++++++++-----
.../oss/pulsar/jms/tracing/TracingUtils.java | 14 +--
2 files changed, 88 insertions(+), 34 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 0af2d244..5023cec9 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -21,7 +21,6 @@
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
@@ -333,6 +332,47 @@ private TraceLevel getTracingLevel(Producer producer) {
}
}
+ private static void addMinimumProducerDetails(
+ Producer producer, Map<String, Object> traceDetails) {
+ if (producer == null) return;
+
+ traceDetails.put("producerId", producer.getProducerId());
+ traceDetails.put("producerName", producer.getProducerName());
+ if (producer.getAccessMode() != null) {
+ traceDetails.put("accessMode", producer.getAccessMode().name());
+ }
+ traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress()));
+ if (producer.getTopic() != null) {
+ traceDetails.put(
+ "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+ }
+ traceDetails.put("authRole", producer.getCnx().getAuthRole());
+ }
+
+ private static void addMinimumConsumerSubscriptionDetails(
+ Consumer consumer, Map<String, Object> traceDetails) {
+ if (consumer == null) return;
+
+ addMinimumConsumerSubscriptionDetails(consumer, consumer.getSubscription(), traceDetails);
+ }
+
+ private static void addMinimumConsumerSubscriptionDetails(
+ Consumer consumer, Subscription subscription, Map<String, Object> traceDetails) {
+ if (consumer != null) {
+ traceDetails.put("consumerName", consumer.consumerName());
+ traceDetails.put("consumerId", consumer.consumerId());
+ traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress()));
+ traceDetails.put("authRole", consumer.cnx().getAuthRole());
+ }
+
+ if (subscription != null) {
+ traceDetails.put("subscriptionName", subscription.getName());
+ traceDetails.put(
+ "topicName", TopicName.get(subscription.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("subscriptionType", subscription.getType().name());
+ }
+ }
+
/* ***************************
** Administrative events
******************************/
@@ -344,6 +384,8 @@ public void onConnectionCreated(ServerCnx cnx) {
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
+
trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
}
@@ -356,6 +398,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
}
@@ -369,6 +412,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
PublisherStatsImpl stats = producer.getStats();
traceDetails.put("connectedSince", stats.getConnectedSince());
@@ -391,6 +435,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
}
@@ -405,9 +450,9 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
ConsumerStatsImpl stats = consumer.getStats();
-
traceDetails.put("connectedSince", stats.getConnectedSince());
traceDetails.put("closedAt", DateFormatter.now());
traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
@@ -417,6 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
traceDetails.put("unackedMessages", stats.getUnackedMessages());
traceDetails.put("messageAckRate", stats.getMessageAckRate());
+ traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
}
@@ -446,6 +492,7 @@ public void onConnectionClosed(ServerCnx cnx) {
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
}
@@ -466,12 +513,12 @@ public void beforeSendMessage(
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("subscription", getSubscriptionDetails(subscription));
- traceDetails.put("consumer", getConsumerDetails(consumer));
+
+ addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
+
traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
- traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
- trace(EventReasons.MESSAGE, "Message read", traceDetails);
+ trace(EventReasons.MESSAGE, "read", traceDetails);
}
public void onMessagePublish(
@@ -483,7 +530,9 @@ public void onMessagePublish(
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("producer", getProducerDetails(producer, traceSchema));
+
+ addMinimumProducerDetails(producer, traceDetails);
+
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
@@ -491,7 +540,7 @@ public void onMessagePublish(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "Message received", traceDetails);
+ trace(EventReasons.MESSAGE, "received", traceDetails);
}
public void messageProduced(
@@ -507,12 +556,12 @@ public void messageProduced(
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
- traceDetails.put("producer", getProducerDetails(producer, traceSchema));
+ addMinimumProducerDetails(producer, traceDetails);
+
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
- trace(EventReasons.MESSAGE, "Message stored", traceDetails);
+ trace(EventReasons.MESSAGE, "stored", traceDetails);
}
public void messageDispatched(
@@ -523,10 +572,7 @@ public void messageDispatched(
if (level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("consumer", getConsumerDetails(consumer));
- if (consumer != null) {
- traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
- }
+ addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
traceDetails.put("messageId", ledgerId + ":" + entryId);
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
@@ -534,31 +580,41 @@ public void messageDispatched(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "Message dispatched", traceDetails);
+ trace(EventReasons.MESSAGE, "dispatched", traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
TraceLevel level = getTracingLevel(consumer);
- if (level == TraceLevel.OFF) return;
+ if (consumer != null && level == TraceLevel.OFF) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("consumer", getConsumerDetails(consumer));
- if (consumer != null) {
- traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+
+ addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
+
+ if (consumer == null) {
+ // ack with empty consumer == message filtered by JMSFilter
+ traceDetails.put("reason", "filtered by JMSFilter");
+ } else {
+ // todo: am I right that unacked/nacked messages never go through broker interceptor?
+ // in this case we need consumer interceptor to track nacks
+ traceDetails.put("reason", "acked");
+ }
+
+ if (consumer != null && consumer.getSubscription() != null) {
+ Subscription sub = consumer.getSubscription();
+ traceDetails.put("subscriptionName", sub.getName());
+ traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+ traceDetails.put("subscriptionType", sub.getType().name());
}
Map<String, Object> ackDetails = new TreeMap<>();
if (ackCmd.hasAckType()) {
ackDetails.put("type", ackCmd.getAckType().name());
- } else {
- ackDetails.put("type", "NOT SET");
}
if (ackCmd.hasConsumerId()) {
- ackDetails.put("consumerId", ackCmd.getConsumerId());
- } else {
- ackDetails.put("consumerId", "NOT SET");
+ ackDetails.put("ackConsumerId", ackCmd.getConsumerId());
}
ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
ackDetails.put(
@@ -579,7 +635,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
traceDetails.put("ack", ackDetails);
- trace(EventReasons.MESSAGE, "Message acked", traceDetails);
+ trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
}
@NotNull
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 25f933e1..f0317a0c 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -475,6 +475,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
}
+ traceDetails.put("state", cnx.getState());
+
AuthenticationDataSource authData = cnx.getAuthenticationData();
if (authData != null) {
traceDetails.put("authData", getAuthDataDetails(authData));
@@ -639,10 +641,6 @@ private static void populateMessageMetadataDetails(
if (msgMetadata.hasSequenceId()) {
traceDetails.put("sequenceId", msgMetadata.getSequenceId());
}
- if (msgMetadata.hasProducerName()) {
- traceDetails.put("producerName", msgMetadata.getProducerName());
- }
-
if (msgMetadata.hasUncompressedSize()) {
traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
}
@@ -703,12 +701,12 @@ private static void populatePublishContext(
traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
traceDetails.put("isChunked", publishContext.isChunked());
traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
-
traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
traceDetails.put("msgSize", publishContext.getMsgSize());
- traceDetails.put("producerName", publishContext.getProducerName());
- traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
- traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+ if (publishContext.getOriginalProducerName() != null) {
+ traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+ traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+ }
traceDetails.put("sequenceId", publishContext.getSequenceId());
}
From c74a95bbe3ac5f8d7f029d499581dbbbc1db850b Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 7 May 2024 18:31:12 -0700
Subject: [PATCH 26/29] tweaks, servlet tracing
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++++-----
.../oss/pulsar/jms/tracing/TracingUtils.java | 76 +++++++++++
2 files changed, 168 insertions(+), 29 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 5023cec9..cc34cf57 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -24,6 +24,7 @@
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
@@ -33,7 +34,6 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import io.netty.buffer.ByteBuf;
-import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
@@ -44,11 +44,13 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.intercept.BrokerInterceptor;
import org.apache.pulsar.broker.service.Consumer;
@@ -83,9 +85,9 @@ public class BrokerTracing implements BrokerInterceptor {
private static void loadEnabledEvents(
PulsarService pulsarService, Set<EventReasons> enabledEvents) {
Properties props = pulsarService.getConfiguration().getProperties();
- if (props.contains("jmsTracingEventList")) {
+ if (props.containsKey("jmsTracingEventList")) {
String events = props.getProperty("jmsTracingEventList", "");
- log.debug("read jmsTracingEventList: {}", events);
+ log.info("read jmsTracingEventList: {}", events);
enabledEvents.clear();
for (String event : events.split(",")) {
@@ -109,7 +111,9 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
.getProperties()
.getProperty("jmsTracingLevel", defaultTraceLevel.toString());
try {
- return TraceLevel.valueOf(level.trim().toUpperCase());
+ TraceLevel traceLevel = TraceLevel.valueOf(level.trim().toUpperCase());
+ log.info("Using tracing level: {}", traceLevel);
+ return traceLevel;
} catch (IllegalArgumentException e) {
log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
return defaultTraceLevel;
@@ -268,6 +272,17 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
});
}
+ @NotNull
+ private static String formatMessageId(MessageIdData x) {
+ String msgId = x.getLedgerId() + ":" + x.getEntryId();
+ if (x.hasBatchIndex()) {
+ msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
+ } else if (x.getAckSetsCount() > 0) {
+ msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
+ }
+ return msgId;
+ }
+
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
@@ -277,12 +292,15 @@ public void initialize(PulsarService pulsarService) {
Properties props = pulsarService.getConfiguration().getProperties();
if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
+ log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength);
}
if (props.containsKey("jmsTracingTraceSystemTopics")) {
traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
+ log.info("Setting traceSystemTopics to {}", traceSystemTopics);
}
if (props.containsKey("jmsTracingTraceSchema")) {
traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
+ log.info("Setting traceSchema to {}", traceSchema);
}
if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
cacheTraceLevelsDurationSec =
@@ -292,6 +310,7 @@ public void initialize(PulsarService pulsarService) {
"Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10);
cacheTraceLevelsDurationSec = 10;
}
+ log.info("Setting cacheTraceLevelsDurationSec to {}", cacheTraceLevelsDurationSec);
}
}
@@ -472,12 +491,47 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
if (traceLevel == TraceLevel.OFF) return;
+ traceCommandInternal(command, cnx);
+
+ // todo: cache topics, and check if system topic
+ // without cache:
+
+ // final String topicName = getCommandTopic(command);
+ // if (topicName == null) {
+ // // don't know if a system topic, trace it
+ // traceCommandInternal(command, cnx);
+ // return;
+ // }
+ //
+ // cnx.getBrokerService()
+ // .getTopicIfExists(topicName)
+ // .whenComplete(
+ // (topic, ex) -> {
+ // if (ex != null) {
+ // log.error("Error getting topic {} to trace command {}", topicName, command,
+ // ex);
+ // return;
+ // }
+ //
+ // // skip system topics if needed
+ // if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic())
+ // return;
+ //
+ // traceCommandInternal(command, cnx);
+ // });
+ }
+
+ private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) {
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
+
+ traceDetails.put("authMethod", cnx.getAuthMethod());
+ traceDetails.put("authRole", cnx.getAuthRole());
+ traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
+ traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
if (command.hasType()) {
- traceDetails.put("type", command.getType().name());
- traceDetails.put("command", getCommandDetails(command));
+ traceDetails.put("command", command.getType().name());
+ traceDetails.put("parameters", getCommandDetails(command));
} else {
traceDetails.put("type", "unknown/null");
}
@@ -638,17 +692,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
}
- @NotNull
- private static String formatMessageId(MessageIdData x) {
- String msgId = x.getLedgerId() + ":" + x.getEntryId();
- if (x.hasBatchIndex()) {
- msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
- } else if (x.getAckSetsCount() > 0) {
- msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
- }
- return msgId;
- }
-
/* ***************************
** Transaction events
******************************/
@@ -679,18 +722,38 @@ public void txnEnded(String txnID, long txnAction) {
** Servlet events
******************************/
- public void onWebserviceRequest(ServletRequest request)
- throws IOException, ServletException, InterceptException {
- // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
- // log.info("onWebserviceRequest: Tracing servlet requests not supported");
- // }
+ public void onWebserviceRequest(ServletRequest request) {
+ // skipping, it is the same as onWebserviceResponse
+ // but without response status.
}
- public void onWebserviceResponse(ServletRequest request, ServletResponse response)
- throws IOException, ServletException {
- // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
- // log.info("onWebserviceResponse: Tracing servlet requests not supported");
- // }
+ public void onWebserviceResponse(ServletRequest request, ServletResponse response) {
+ if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return;
+ if (traceLevel == TraceLevel.OFF) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+
+ traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost()));
+ traceDetails.put("protocol", request.getProtocol());
+ traceDetails.put("scheme", request.getScheme());
+
+ try {
+ HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true);
+ traceDetails.put("method", req.getMethod());
+ traceDetails.put("uri", req.getRequestURI());
+ if (req.getQueryString() != null) {
+ traceDetails.put("queryString", req.getQueryString());
+ }
+ traceDetails.put("authType", req.getAuthType());
+ traceDetails.put("remoteUser", req.getRemoteUser());
+
+ HttpServletResponse resp = (HttpServletResponse) response;
+ traceDetails.put("status", resp.getStatus());
+ } catch (Throwable t) {
+ log.error("Error getting request details", t);
+ }
+
+ trace(EventReasons.SERVLET, "WebService response", traceDetails);
}
// not needed
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index f0317a0c..136ad5fd 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -360,6 +360,82 @@ private static void populateCommandDetails(
}
}
+ public static String getCommandTopic(BaseCommand command) {
+ if (command == null) {
+ return null;
+ }
+
+ if (!command.hasType()) {
+ return null;
+ }
+
+ // Currently not doing transitive topic resolution by e.g. topic pattern
+ // or by producerId/consumerId to the topic they are using.
+ // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by
+ // requestId.
+ switch (command.getType()) {
+ case SUBSCRIBE:
+ if (command.getSubscribe().hasTopic()) {
+ return command.getSubscribe().getTopic();
+ }
+ break;
+ case PRODUCER:
+ if (command.getProducer().hasTopic()) {
+ return command.getProducer().getTopic();
+ }
+ break;
+ case PARTITIONED_METADATA:
+ if (command.getPartitionMetadata().hasTopic()) {
+ return command.getPartitionMetadata().getTopic();
+ }
+ break;
+ case LOOKUP:
+ if (command.getLookupTopic().hasTopic()) {
+ return command.getLookupTopic().getTopic();
+ }
+ break;
+ case GET_SCHEMA:
+ if (command.getGetSchema().hasTopic()) {
+ return command.getGetSchema().getTopic();
+ }
+ break;
+ case GET_OR_CREATE_SCHEMA:
+ if (command.getGetOrCreateSchema().hasTopic()) {
+ return command.getGetOrCreateSchema().getTopic();
+ }
+ break;
+ case ADD_SUBSCRIPTION_TO_TXN:
+ if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0
+ && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) {
+ Optional<org.apache.pulsar.common.api.proto.Subscription> subscription =
+ command
+ .getAddSubscriptionToTxn()
+ .getSubscriptionsList()
+ .stream()
+ .filter(sub -> sub.hasTopic())
+ .findFirst();
+ if (subscription.isPresent()) {
+ return subscription.get().getTopic();
+ }
+ }
+ break;
+ case END_TXN_ON_PARTITION:
+ if (command.getEndTxnOnPartition().hasTopic()) {
+ return command.getEndTxnOnPartition().getTopic();
+ }
+ break;
+ case END_TXN_ON_SUBSCRIPTION:
+ if (command.getEndTxnOnSubscription().hasSubscription()
+ && command.getEndTxnOnSubscription().getSubscription().hasTopic()) {
+ return command.getEndTxnOnSubscription().getSubscription().getTopic();
+ }
+ break;
+ default:
+ return null;
+ }
+ return null;
+ }
+
private static final Set<String> skipTraceFields =
Sets.newHashSet(
"authdata",
From 70a82888827f8e98a16a1bed682c0dc70a83ba7e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 8 May 2024 14:24:48 -0700
Subject: [PATCH 27/29] jmsTracingLevel as a master off switch; reworked
categories, renamed config param
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 266 +++++------
.../oss/pulsar/jms/tracing/TracingUtils.java | 440 ++----------------
.../pulsar/jms/tracing/TracingUtilsTest.java | 12 +-
3 files changed, 172 insertions(+), 546 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index cc34cf57..dafa8ee0 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,9 +15,9 @@
*/
package com.datastax.oss.pulsar.jms.tracing;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventCategory;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventSubCategory;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
@@ -34,7 +34,6 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import io.netty.buffer.ByteBuf;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
@@ -63,7 +62,7 @@
import org.apache.pulsar.common.api.proto.CommandAck;
import org.apache.pulsar.common.api.proto.MessageIdData;
import org.apache.pulsar.common.api.proto.MessageMetadata;
-import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.api.proto.TxnAction;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
@@ -75,7 +74,7 @@ public class BrokerTracing implements BrokerInterceptor {
private static final TraceLevel defaultTraceLevel = TraceLevel.OFF;
- private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
+ private final Set<EventCategory> jmsTracingEventCategory = new HashSet<>();
private TraceLevel traceLevel = defaultTraceLevel;
private int maxBinaryDataLength = 256;
private int cacheTraceLevelsDurationSec = 10;
@@ -83,23 +82,30 @@ public class BrokerTracing implements BrokerInterceptor {
private boolean traceSchema = false;
private static void loadEnabledEvents(
- PulsarService pulsarService, Set<EventReasons> enabledEvents) {
+ PulsarService pulsarService, Set<EventCategory> enabledEvents, TraceLevel traceLevel) {
+
+ if (traceLevel == TraceLevel.OFF) {
+ log.info("Tracing is disabled. jmsTracingEventCategory is ignored.");
+ enabledEvents.clear();
+ return;
+ }
+
Properties props = pulsarService.getConfiguration().getProperties();
- if (props.containsKey("jmsTracingEventList")) {
- String events = props.getProperty("jmsTracingEventList", "");
- log.info("read jmsTracingEventList: {}", events);
+ if (props.containsKey("jmsTracingEventCategory")) {
+ String events = props.getProperty("jmsTracingEventCategory", "");
+ log.info("read jmsTracingEventCategory: {}", events);
enabledEvents.clear();
for (String event : events.split(",")) {
try {
- enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
+ enabledEvents.add(EventCategory.valueOf(event.trim().toUpperCase()));
} catch (IllegalArgumentException e) {
log.error("Invalid event: {}. Skipping", event);
}
}
} else {
- log.warn("jmsTracingEventList not set. Using all available.");
- enabledEvents.addAll(Arrays.asList(EventReasons.values()));
+ log.warn("jmsTracingEventCategory is not set. Using ADMIN, CONN.");
+ enabledEvents.add(EventCategory.CONN);
}
}
@@ -286,9 +292,10 @@ private static String formatMessageId(MessageIdData x) {
public void initialize(PulsarService pulsarService) {
log.info("Initializing BrokerTracing");
- loadEnabledEvents(pulsarService, jmsTracingEventList);
traceLevel = getTraceLevel(pulsarService);
+ loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel);
+
Properties props = pulsarService.getConfiguration().getProperties();
if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
@@ -393,41 +400,49 @@ private static void addMinimumConsumerSubscriptionDetails(
}
/* ***************************
- ** Administrative events
+ ** Connection events
******************************/
public void onConnectionCreated(ServerCnx cnx) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+ traceDetails.put("serverCnx", getConnectionDetails(cnx));
+ traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
- if (traceLevel == TraceLevel.OFF) return;
+ trace(EventCategory.CONN, EventSubCategory.CREATED, traceDetails);
+ }
+
+ public void onConnectionClosed(ServerCnx cnx) {
+ if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return;
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("serverCnx", getConnectionDetails(cnx));
traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
- trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
+ trace(EventCategory.CONN, EventSubCategory.CLOSED, traceDetails);
}
+ /* **************************
+ * Producer connection events
+ *****************************/
+
public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return;
if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.OFF) return;
-
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
- trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
+ trace(EventCategory.PROD, EventSubCategory.CREATED, traceDetails);
}
public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return;
if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.OFF) return;
-
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("producer", getProducerDetails(producer, traceSchema));
traceDetails.put("metadata", metadata);
@@ -441,30 +456,30 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
// no message count in stats? stats.getCount() is not it
- trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
+ trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails);
}
+ /* **************************
+ * Consumer connection events
+ *****************************/
+
public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return;
if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.OFF) return;
-
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
traceDetails.put("metadata", metadata);
traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
- trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
+ trace(EventCategory.CONS, EventSubCategory.CREATED, traceDetails);
}
public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return;
if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
- if (traceLevel == TraceLevel.OFF) return;
-
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("consumer", getConsumerDetails(consumer));
traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
@@ -483,102 +498,16 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
traceDetails.put("messageAckRate", stats.getMessageAckRate());
traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
- trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
- }
-
- public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
- if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
-
- if (traceLevel == TraceLevel.OFF) return;
-
- traceCommandInternal(command, cnx);
-
- // todo: cache topics, and check if system topic
- // without cache:
-
- // final String topicName = getCommandTopic(command);
- // if (topicName == null) {
- // // don't know if a system topic, trace it
- // traceCommandInternal(command, cnx);
- // return;
- // }
- //
- // cnx.getBrokerService()
- // .getTopicIfExists(topicName)
- // .whenComplete(
- // (topic, ex) -> {
- // if (ex != null) {
- // log.error("Error getting topic {} to trace command {}", topicName, command,
- // ex);
- // return;
- // }
- //
- // // skip system topics if needed
- // if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic())
- // return;
- //
- // traceCommandInternal(command, cnx);
- // });
- }
-
- private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) {
- Map<String, Object> traceDetails = new TreeMap<>();
-
- traceDetails.put("authMethod", cnx.getAuthMethod());
- traceDetails.put("authRole", cnx.getAuthRole());
- traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
- traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
-
- if (command.hasType()) {
- traceDetails.put("command", command.getType().name());
- traceDetails.put("parameters", getCommandDetails(command));
- } else {
- traceDetails.put("type", "unknown/null");
- }
-
- trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails);
- }
-
- public void onConnectionClosed(ServerCnx cnx) {
- if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
-
- if (traceLevel == TraceLevel.OFF) return;
-
- Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("serverCnx", getConnectionDetails(cnx));
- traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
-
- trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
+ trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails);
}
/* ***************************
** Message events
******************************/
- public void beforeSendMessage(
- Subscription subscription,
- Entry entry,
- long[] ackSet,
- MessageMetadata msgMetadata,
- Consumer consumer) {
- if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
-
- TraceLevel level = getTracingLevel(subscription);
- if (level == TraceLevel.OFF) return;
-
- Map<String, Object> traceDetails = new TreeMap<>();
-
- addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
-
- traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
-
- trace(EventReasons.MESSAGE, "read", traceDetails);
- }
-
public void onMessagePublish(
Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
-
- if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.OFF) return;
@@ -594,7 +523,7 @@ public void onMessagePublish(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "received", traceDetails);
+ trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
}
public void messageProduced(
@@ -604,7 +533,7 @@ public void messageProduced(
long ledgerId,
long entryId,
Topic.PublishContext publishContext) {
- if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
TraceLevel level = getTracingLevel(producer);
if (level == TraceLevel.OFF) return;
@@ -615,12 +544,32 @@ public void messageProduced(
traceDetails.put("publishContext", getPublishContextDetails(publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
traceDetails.put("startTimeNs", startTimeNs);
- trace(EventReasons.MESSAGE, "stored", traceDetails);
+ trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails);
+ }
+
+ public void beforeSendMessage(
+ Subscription subscription,
+ Entry entry,
+ long[] ackSet,
+ MessageMetadata msgMetadata,
+ Consumer consumer) {
+ if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
+
+ TraceLevel level = getTracingLevel(subscription);
+ if (level == TraceLevel.OFF) return;
+
+ Map<String, Object> traceDetails = new TreeMap<>();
+
+ addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
+
+ traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+
+ trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
}
public void messageDispatched(
ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
- if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
TraceLevel level = getTracingLevel(consumer);
if (level == TraceLevel.OFF) return;
@@ -634,15 +583,17 @@ public void messageDispatched(
"headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
traceDetails.put("payload", headersAndPayloadDetails);
- trace(EventReasons.MESSAGE, "dispatched", traceDetails);
+ trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
}
public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
- if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
TraceLevel level = getTracingLevel(consumer);
if (consumer != null && level == TraceLevel.OFF) return;
+ EventSubCategory subcategory = EventSubCategory.ACKED;
+
Map<String, Object> traceDetails = new TreeMap<>();
addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
@@ -650,6 +601,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
if (consumer == null) {
// ack with empty consumer == message filtered by JMSFilter
traceDetails.put("reason", "filtered by JMSFilter");
+ subcategory = EventSubCategory.FILTERED;
} else {
// todo: am I right that unacked/nacked messages never go through broker interceptor?
// in this case we need consumer interceptor to track nacks
@@ -662,7 +614,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
traceDetails.put("subscriptionType", sub.getType().name());
}
-
Map<String, Object> ackDetails = new TreeMap<>();
if (ackCmd.hasAckType()) {
ackDetails.put("type", ackCmd.getAckType().name());
@@ -689,7 +640,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
traceDetails.put("ack", ackDetails);
- trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
+ trace(EventCategory.MSG, subcategory, traceDetails);
}
/* ***************************
@@ -697,39 +648,51 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
******************************/
public void txnOpened(long tcId, String txnID) {
- if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
- if (traceLevel == TraceLevel.OFF) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.TX)) return;
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("tcId", tcId);
+ traceDetails.put("tcId", tcId); // transaction coordinator id
traceDetails.put("txnID", txnID);
- trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails);
+ trace(EventCategory.TX, EventSubCategory.OPENED, traceDetails);
}
public void txnEnded(String txnID, long txnAction) {
- if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
- if (traceLevel == TraceLevel.OFF) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.TX)) return;
+ final EventSubCategory subcategory;
Map<String, Object> traceDetails = new TreeMap<>();
traceDetails.put("txnID", txnID);
- traceDetails.put("txnAction", txnAction);
- trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails);
+ TxnAction action = TxnAction.valueOf((int) txnAction);
+ if (action == null) {
+ subcategory = EventSubCategory.CLOSED;
+ traceDetails.put("txnAction", "unknown action code " + txnAction);
+ } else {
+ traceDetails.put("txnAction", action.name());
+ switch (action) {
+ case COMMIT:
+ subcategory = EventSubCategory.COMMITTED;
+ break;
+ case ABORT:
+ subcategory = EventSubCategory.ABORTED;
+ break;
+ default:
+ subcategory = EventSubCategory.CLOSED;
+ traceDetails.put("txnAction", "unknown action code " + txnAction + " " + action.name());
+ break;
+ }
+ }
+
+ trace(EventCategory.TX, subcategory, traceDetails);
}
/* ***************************
** Servlet events
******************************/
- public void onWebserviceRequest(ServletRequest request) {
- // skipping, it is the same as onWebserviceResponse
- // but without response status.
- }
-
public void onWebserviceResponse(ServletRequest request, ServletResponse response) {
- if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return;
- if (traceLevel == TraceLevel.OFF) return;
+ if (!jmsTracingEventCategory.contains(EventCategory.REST)) return;
Map<String, Object> traceDetails = new TreeMap<>();
@@ -737,6 +700,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
traceDetails.put("protocol", request.getProtocol());
traceDetails.put("scheme", request.getScheme());
+ // todo: log POST payload?
try {
HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true);
traceDetails.put("method", req.getMethod());
@@ -753,9 +717,21 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
log.error("Error getting request details", t);
}
- trace(EventReasons.SERVLET, "WebService response", traceDetails);
+ trace(EventCategory.REST, EventSubCategory.CALLED, traceDetails);
+ }
+
+ /* ***************************
+ ** Skipped
+ ******************************/
+
+ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) {
+ // skipping, output is not useful
+ }
+
+ public void onWebserviceRequest(ServletRequest request) {
+ // skipping, it is the same as onWebserviceResponse
+ // but without response status.
}
- // not needed
// public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain)
}
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 136ad5fd..f8caf8ef 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -21,18 +21,13 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
-import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -45,7 +40,6 @@
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.Subscription;
import org.apache.pulsar.broker.service.Topic;
-import org.apache.pulsar.common.api.proto.BaseCommand;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.compression.CompressionCodec;
import org.apache.pulsar.common.compression.CompressionCodecProvider;
@@ -56,33 +50,53 @@
@Slf4j
public class TracingUtils {
- public enum EventReasons {
- ADMINISTRATIVE,
- COMMANDS,
- MESSAGE,
- TRANSACTION,
- SERVLET,
+ public enum EventCategory {
+ CONN, // connection creation, closure,
+ PROD, // producer creation, closure,
+ CONS, // consumer creation, closure,
+ TX, // (transaction creation, commit, rollback,etc),
+ MSG, // (message level send,dispatch,ack,expire,acktimeout, negative ack, etc),
+ REST, // (rest api calls),
+ }
+
+ public enum EventSubCategory {
+ CREATED,
+ CLOSED,
+
+ PRODUCED,
+ STORED,
+
+ READ,
+ DISPATCHED,
+ ACKED,
+ FILTERED,
+
+ OPENED,
+ COMMITTED,
+ ABORTED,
+
+ CALLED,
}
@FunctionalInterface
public interface Tracer {
- void trace(EventReasons reason, String message);
+ void trace(EventCategory category, String message);
}
public static class Slf4jTracer implements Tracer {
- private static final Map<EventReasons, org.slf4j.Logger> traceLoggers = new HashMap<>();
+ private static final Map<EventCategory, org.slf4j.Logger> traceLoggers = new HashMap<>();
static {
- for (EventReasons reason : EventReasons.values()) {
+ for (EventCategory category : EventCategory.values()) {
traceLoggers.put(
- reason,
- org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase()));
+ category,
+ org.slf4j.LoggerFactory.getLogger("jms-tracing-" + category.name().toLowerCase()));
}
}
@Override
- public void trace(EventReasons reason, String message) {
- traceLoggers.get(reason).info(message);
+ public void trace(EventCategory category, String message) {
+ traceLoggers.get(category).info(message);
}
}
@@ -136,397 +150,33 @@ public static String hostNameOf(String clientAddress) {
}
}
- public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
- trace(SLF4J_TRACER, reason, message, traceDetails);
+ public static void trace(
+ EventCategory category, EventSubCategory subCategory, Map<String, Object> traceDetails) {
+ trace(SLF4J_TRACER, category, subCategory, traceDetails);
}
public static void trace(
- Tracer tracer, EventReasons reason, String message, Map<String, Object> traceDetails) {
+ Tracer tracer,
+ EventCategory category,
+ EventSubCategory subCategory,
+ Map<String, Object> traceDetails) {
Map<String, Object> trace = new TreeMap<>();
- trace.put("eventType", message);
+ trace.put("event", category + "_" + subCategory);
trace.put("traceDetails", traceDetails);
try {
String loggableJsonString = mapper.writeValueAsString(trace);
- tracer.trace(reason, loggableJsonString);
+ tracer.trace(category, loggableJsonString);
} catch (JsonProcessingException e) {
log.error(
- "Failed to serialize trace event type '{}' as json, traceDetails: {}",
- message,
+ "Failed to serialize trace event '{}_{}' as json, traceDetails: {}",
+ category,
+ subCategory,
traceDetails,
e);
}
}
- public static Map<String, Object> getCommandDetails(BaseCommand command) {
- if (command == null) {
- return null;
- }
-
- Map<String, Object> details = new TreeMap<>();
- populateCommandDetails(command, details);
- return details;
- }
-
- private static void populateCommandDetails(
- BaseCommand command, Map<String, Object> traceDetails) {
- if (command == null) {
- return;
- }
-
- if (!command.hasType()) {
- return;
- }
-
- // trace all params otherwise
- switch (command.getType()) {
- case CONNECT:
- populateByReflection(command.getConnect(), traceDetails);
- break;
- case CONNECTED:
- populateByReflection(command.getConnected(), traceDetails);
- break;
- case SUBSCRIBE:
- populateByReflection(command.getSubscribe(), traceDetails);
- break;
- case PRODUCER:
- populateByReflection(command.getProducer(), traceDetails);
- break;
- case SEND:
- populateByReflection(command.getSend(), traceDetails);
- break;
- case SEND_RECEIPT:
- populateByReflection(command.getSendReceipt(), traceDetails);
- break;
- case SEND_ERROR:
- populateByReflection(command.getSendError(), traceDetails);
- break;
- case MESSAGE:
- populateByReflection(command.getMessage(), traceDetails);
- break;
- case ACK:
- populateByReflection(command.getAck(), traceDetails);
- break;
- case FLOW:
- populateByReflection(command.getFlow(), traceDetails);
- break;
- case UNSUBSCRIBE:
- populateByReflection(command.getUnsubscribe(), traceDetails);
- break;
- case SUCCESS:
- populateByReflection(command.getSuccess(), traceDetails);
- break;
- case ERROR:
- populateByReflection(command.getError(), traceDetails);
- break;
- case CLOSE_PRODUCER:
- populateByReflection(command.getCloseProducer(), traceDetails);
- break;
- case CLOSE_CONSUMER:
- populateByReflection(command.getCloseConsumer(), traceDetails);
- break;
- case PRODUCER_SUCCESS:
- populateByReflection(command.getProducerSuccess(), traceDetails);
- break;
- case PING:
- populateByReflection(command.getPing(), traceDetails);
- break;
- case PONG:
- populateByReflection(command.getPong(), traceDetails);
- break;
- case REDELIVER_UNACKNOWLEDGED_MESSAGES:
- populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
- break;
- case PARTITIONED_METADATA:
- populateByReflection(command.getPartitionMetadata(), traceDetails);
- break;
- case PARTITIONED_METADATA_RESPONSE:
- populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
- break;
- case LOOKUP:
- populateByReflection(command.getLookupTopic(), traceDetails);
- break;
- case LOOKUP_RESPONSE:
- populateByReflection(command.getLookupTopicResponse(), traceDetails);
- break;
- case CONSUMER_STATS:
- populateByReflection(command.getConsumerStats(), traceDetails);
- break;
- case CONSUMER_STATS_RESPONSE:
- populateByReflection(command.getConsumerStatsResponse(), traceDetails);
- break;
- case REACHED_END_OF_TOPIC:
- populateByReflection(command.getReachedEndOfTopic(), traceDetails);
- break;
- case SEEK:
- populateByReflection(command.getSeek(), traceDetails);
- break;
- case GET_LAST_MESSAGE_ID:
- populateByReflection(command.getGetLastMessageId(), traceDetails);
- break;
- case GET_LAST_MESSAGE_ID_RESPONSE:
- populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
- break;
- case ACTIVE_CONSUMER_CHANGE:
- populateByReflection(command.getActiveConsumerChange(), traceDetails);
- break;
- case GET_TOPICS_OF_NAMESPACE:
- populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
- break;
- case GET_TOPICS_OF_NAMESPACE_RESPONSE:
- populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
- break;
- case GET_SCHEMA:
- populateByReflection(command.getGetSchema(), traceDetails);
- break;
- case GET_SCHEMA_RESPONSE:
- populateByReflection(command.getGetSchemaResponse(), traceDetails);
- break;
- case AUTH_CHALLENGE:
- populateByReflection(command.getAuthChallenge(), traceDetails);
- break;
- case AUTH_RESPONSE:
- populateByReflection(command.getAuthResponse(), traceDetails);
- break;
- case ACK_RESPONSE:
- populateByReflection(command.getAckResponse(), traceDetails);
- break;
- case GET_OR_CREATE_SCHEMA:
- populateByReflection(command.getGetOrCreateSchema(), traceDetails);
- break;
- case GET_OR_CREATE_SCHEMA_RESPONSE:
- populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
- break;
- case NEW_TXN:
- populateByReflection(command.getNewTxn(), traceDetails);
- break;
- case NEW_TXN_RESPONSE:
- populateByReflection(command.getNewTxnResponse(), traceDetails);
- break;
- case ADD_PARTITION_TO_TXN:
- populateByReflection(command.getAddPartitionToTxn(), traceDetails);
- break;
- case ADD_PARTITION_TO_TXN_RESPONSE:
- populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
- break;
- case ADD_SUBSCRIPTION_TO_TXN:
- populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
- break;
- case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
- populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
- break;
- case END_TXN:
- populateByReflection(command.getEndTxn(), traceDetails);
- break;
- case END_TXN_RESPONSE:
- populateByReflection(command.getEndTxnResponse(), traceDetails);
- break;
- case END_TXN_ON_PARTITION:
- populateByReflection(command.getEndTxnOnPartition(), traceDetails);
- break;
- case END_TXN_ON_PARTITION_RESPONSE:
- populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
- break;
- case END_TXN_ON_SUBSCRIPTION:
- populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
- break;
- case END_TXN_ON_SUBSCRIPTION_RESPONSE:
- populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
- break;
- case TC_CLIENT_CONNECT_REQUEST:
- populateByReflection(command.getTcClientConnectRequest(), traceDetails);
- break;
- case TC_CLIENT_CONNECT_RESPONSE:
- populateByReflection(command.getTcClientConnectResponse(), traceDetails);
- break;
- case WATCH_TOPIC_LIST:
- populateByReflection(command.getWatchTopicList(), traceDetails);
- break;
- case WATCH_TOPIC_LIST_SUCCESS:
- populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
- break;
- case WATCH_TOPIC_UPDATE:
- populateByReflection(command.getWatchTopicUpdate(), traceDetails);
- break;
- case WATCH_TOPIC_LIST_CLOSE:
- populateByReflection(command.getWatchTopicListClose(), traceDetails);
- break;
- case TOPIC_MIGRATED:
- populateByReflection(command.getTopicMigrated(), traceDetails);
- break;
- default:
- log.error("Unknown command type: {}", command.getType());
- traceDetails.put("error", "unknownCommandType " + command.getType());
- }
- }
-
- public static String getCommandTopic(BaseCommand command) {
- if (command == null) {
- return null;
- }
-
- if (!command.hasType()) {
- return null;
- }
-
- // Currently not doing transitive topic resolution by e.g. topic pattern
- // or by producerId/consumerId to the topic they are using.
- // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by
- // requestId.
- switch (command.getType()) {
- case SUBSCRIBE:
- if (command.getSubscribe().hasTopic()) {
- return command.getSubscribe().getTopic();
- }
- break;
- case PRODUCER:
- if (command.getProducer().hasTopic()) {
- return command.getProducer().getTopic();
- }
- break;
- case PARTITIONED_METADATA:
- if (command.getPartitionMetadata().hasTopic()) {
- return command.getPartitionMetadata().getTopic();
- }
- break;
- case LOOKUP:
- if (command.getLookupTopic().hasTopic()) {
- return command.getLookupTopic().getTopic();
- }
- break;
- case GET_SCHEMA:
- if (command.getGetSchema().hasTopic()) {
- return command.getGetSchema().getTopic();
- }
- break;
- case GET_OR_CREATE_SCHEMA:
- if (command.getGetOrCreateSchema().hasTopic()) {
- return command.getGetOrCreateSchema().getTopic();
- }
- break;
- case ADD_SUBSCRIPTION_TO_TXN:
- if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0
- && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) {
- Optional<org.apache.pulsar.common.api.proto.Subscription> subscription =
- command
- .getAddSubscriptionToTxn()
- .getSubscriptionsList()
- .stream()
- .filter(sub -> sub.hasTopic())
- .findFirst();
- if (subscription.isPresent()) {
- return subscription.get().getTopic();
- }
- }
- break;
- case END_TXN_ON_PARTITION:
- if (command.getEndTxnOnPartition().hasTopic()) {
- return command.getEndTxnOnPartition().getTopic();
- }
- break;
- case END_TXN_ON_SUBSCRIPTION:
- if (command.getEndTxnOnSubscription().hasSubscription()
- && command.getEndTxnOnSubscription().getSubscription().hasTopic()) {
- return command.getEndTxnOnSubscription().getSubscription().getTopic();
- }
- break;
- default:
- return null;
- }
- return null;
- }
-
- private static final Set<String> skipTraceFields =
- Sets.newHashSet(
- "authdata",
- "authmethod",
- "authmethodname",
- "originalauthdata",
- "orginalauthmethod",
- "originalprincipal",
- "schema");
-
- private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
- if (command == null) {
- return;
- }
- if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) {
- return;
- }
-
- Method[] allMethods = command.getClass().getMethods();
-
- Arrays.stream(allMethods)
- .filter(
- method -> {
- if (!method.getName().startsWith("has")) {
- return false;
- }
- String fieldName = method.getName().substring(3);
- return !skipTraceFields.contains(fieldName.toLowerCase());
- })
- .filter(
- method -> {
- try {
- return (boolean) method.invoke(command);
- } catch (Exception e) {
- return false;
- }
- })
- .forEach(
- method -> {
- String fieldName = method.getName().substring(3);
- try {
- Optional<Method> accessor =
- Arrays.stream(allMethods)
- .filter(
- m ->
- m.getName().equals("get" + fieldName)
- || m.getName().equals("is" + fieldName))
- .findFirst();
- if (!accessor.isPresent()) {
- log.warn(
- "No accessor found for field (but has.. counterpart was found): {} of {}",
- fieldName,
- command.getClass().getCanonicalName());
- return;
- }
- Object value = accessor.get().invoke(command);
-
- if (value == null) return;
-
- // skip logging of binary data
- if (value instanceof byte[]
- || value instanceof ByteBuf
- || value instanceof ByteBuffer) {
- final int size;
- if (value instanceof byte[]) {
- size = ((byte[]) value).length;
- } else if (value instanceof ByteBuf) {
- size = ((ByteBuf) value).readableBytes();
- } else {
- size = ((ByteBuffer) value).remaining();
- }
- traceDetails.put(fieldName + "_size", size);
- return;
- }
-
- if (value
- .getClass()
- .getCanonicalName()
- .contains("org.apache.pulsar.common.api.proto")) {
- Map<String, Object> details = new TreeMap<>();
- populateByReflection(value, details);
- traceDetails.put(fieldName, details);
- } else {
- traceDetails.put(fieldName, value);
- }
- } catch (Exception e) {
- log.error("Failed to access field: {}", fieldName, e);
- }
- });
- }
-
public static Map<String, Object> getConnectionDetails(ServerCnx cnx) {
if (cnx == null) {
return null;
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 71e01bba..9fa3e0d4 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -32,33 +32,33 @@ class TracingUtilsTest {
private Tracer mockTracer =
new Tracer() {
@Override
- public void trace(EventReasons reason, String msg) {
+ public void trace(EventCategory reason, String msg) {
traces.add(msg);
}
};
- private static void trace(Tracer mockTracer, String msg, Map<String, Object> traceDetails) {
- TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails);
+ private static void trace(Tracer mockTracer, Map<String, Object> traceDetails) {
+ TracingUtils.trace(mockTracer, EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
}
@Test
void traceTest() {
traces.clear();
- trace(mockTracer, "msg", null);
+ trace(mockTracer, null);
assertEquals(1, traces.size());
assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
Map<String, Object> map = new TreeMap<>();
traces.clear();
- trace(mockTracer, "msg", map);
+ trace(mockTracer, map);
assertEquals(1, traces.size());
assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
map.put("key1", "value1");
traces.clear();
- trace(mockTracer, "msg", map);
+ trace(mockTracer, map);
assertEquals(1, traces.size());
assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
}
From a447548495ffd574e2a341e069e1f4678bce04b4 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 8 May 2024 17:52:28 -0700
Subject: [PATCH 28/29] PAYLOAD trace levele for messages, port for the host,
tweaks for details
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 52 +++---
.../oss/pulsar/jms/tracing/TracingUtils.java | 153 +++++++++++-------
.../pulsar/jms/tracing/TracingUtilsTest.java | 34 ++--
3 files changed, 152 insertions(+), 87 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dafa8ee0..cfcffa90 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -26,7 +26,7 @@
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf;
import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceMetadataAndPayload;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -76,7 +76,7 @@ public class BrokerTracing implements BrokerInterceptor {
private final Set<EventCategory> jmsTracingEventCategory = new HashSet<>();
private TraceLevel traceLevel = defaultTraceLevel;
- private int maxBinaryDataLength = 256;
+ private int maxPayloadLength = 256;
private int cacheTraceLevelsDurationSec = 10;
private boolean traceSystemTopics = false;
private boolean traceSchema = false;
@@ -297,9 +297,9 @@ public void initialize(PulsarService pulsarService) {
loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel);
Properties props = pulsarService.getConfiguration().getProperties();
- if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
- maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
- log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength);
+ if (props.containsKey("jmsTracingMaxPayloadLength")) {
+ maxPayloadLength = Integer.parseInt(props.getProperty("jmsTracingMaxPayloadLength"));
+ log.info("Setting maxPayloadLength to {}", maxPayloadLength);
}
if (props.containsKey("jmsTracingTraceSystemTopics")) {
traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
@@ -367,7 +367,9 @@ private static void addMinimumProducerDetails(
if (producer.getAccessMode() != null) {
traceDetails.put("accessMode", producer.getAccessMode().name());
}
- traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress()));
+ traceDetails.put("clientHost",
+ TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+
if (producer.getTopic() != null) {
traceDetails.put(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
@@ -387,7 +389,8 @@ private static void addMinimumConsumerSubscriptionDetails(
if (consumer != null) {
traceDetails.put("consumerName", consumer.consumerName());
traceDetails.put("consumerId", consumer.consumerId());
- traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress()));
+ traceDetails.put("clientHost",
+ TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
traceDetails.put("authRole", consumer.cnx().getAuthRole());
}
@@ -516,12 +519,17 @@ public void onMessagePublish(
addMinimumProducerDetails(producer, traceDetails);
- traceDetails.put("publishContext", getPublishContextDetails(publishContext));
+ traceDetails.put("publishContext", getPublishContextDetails(level, publishContext));
- Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
- traceByteBuf(
- "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
- traceDetails.put("payload", headersAndPayloadDetails);
+ if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
+ Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+ traceMetadataAndPayload(
+ "headersAndPayload",
+ headersAndPayload.slice(),
+ headersAndPayloadDetails,
+ maxPayloadLength);
+ traceDetails.put("payload", headersAndPayloadDetails);
+ }
trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
}
@@ -541,9 +549,8 @@ public void messageProduced(
Map<String, Object> traceDetails = new TreeMap<>();
addMinimumProducerDetails(producer, traceDetails);
- traceDetails.put("publishContext", getPublishContextDetails(publishContext));
+ traceDetails.put("publishContext", getPublishContextDetails(level, publishContext));
traceDetails.put("messageId", ledgerId + ":" + entryId);
- traceDetails.put("startTimeNs", startTimeNs);
trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails);
}
@@ -562,7 +569,7 @@ public void beforeSendMessage(
addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
- traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+ traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength));
trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
}
@@ -578,10 +585,15 @@ public void messageDispatched(
addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
traceDetails.put("messageId", ledgerId + ":" + entryId);
- Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
- traceByteBuf(
- "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
- traceDetails.put("payload", headersAndPayloadDetails);
+ if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
+ Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+ traceMetadataAndPayload(
+ "headersAndPayload",
+ headersAndPayload.slice(),
+ headersAndPayloadDetails,
+ maxPayloadLength);
+ traceDetails.put("payload", headersAndPayloadDetails);
+ }
trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
}
@@ -696,7 +708,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost()));
+ traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
traceDetails.put("protocol", request.getProtocol());
traceDetails.put("scheme", request.getScheme());
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index f8caf8ef..4250e867 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -108,8 +108,9 @@ public void trace(EventCategory category, String message) {
.enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
public enum TraceLevel {
- OFF,
- ON
+ OFF, // disabled
+ ON, // enabled without payload tracing
+ PAYLOAD, // enabled with payload tracing
}
private static final LoadingCache<String, String> ipResolverCache =
@@ -137,13 +138,28 @@ public String load(String clientAddress) {
}
});
- public static String hostNameOf(String clientAddress) {
+ public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) {
+ if (clientAddress == null || clientAddress.isEmpty()
+ || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) {
+ return "unknown/null";
+ }
+
+ try {
+ String port = clientSourceAddressAndPort.split(":")[1];
+ return ipResolverCache.get(clientAddress) + ":" + port;
+ } catch (Throwable t) {
+ log.error("Failed to resolve DNS for {}", clientAddress, t);
+ return clientAddress;
+ }
+ }
+
+ public static String hostNameOf(String clientAddress, int remotePort) {
if (clientAddress == null || clientAddress.isEmpty()) {
return "unknown/null";
}
try {
- return ipResolverCache.get(clientAddress);
+ return ipResolverCache.get(clientAddress) + ":" + remotePort;
} catch (Throwable t) {
log.error("Failed to resolve DNS for {}", clientAddress, t);
return clientAddress;
@@ -191,9 +207,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
if (cnx == null) {
return;
}
- traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
+ traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
traceDetails.put("authRole", cnx.getAuthRole());
- traceDetails.put("principal", cnx.getPrincipal());
traceDetails.put("clientVersion", cnx.getClientVersion());
traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
traceDetails.put("authMethod", cnx.getAuthMethod());
@@ -293,7 +308,7 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
traceDetails.put("priorityLevel", consumer.getPriorityLevel());
traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
- traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress()));
+ traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
traceDetails.put("metadata", consumer.getMetadata());
traceDetails.put("unackedMessages", consumer.getUnackedMessages());
@@ -325,7 +340,7 @@ private static void populateProducerDetails(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
}
- traceDetails.put("clientHost", hostNameOf(producer.getClientAddress()));
+ traceDetails.put("clientHost", hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
traceDetails.put("metadata", producer.getMetadata());
@@ -389,18 +404,19 @@ private static void populateMessageMetadataDetails(
}
}
- public static Map<String, Object> getEntryDetails(Entry entry, int maxBinaryDataLength) {
+ public static Map<String, Object> getEntryDetails(
+ TraceLevel level, Entry entry, int maxBinaryDataLength) {
if (entry == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populateEntryDetails(entry, details, maxBinaryDataLength);
+ populateEntryDetails(level, entry, details, maxBinaryDataLength);
return details;
}
private static void populateEntryDetails(
- Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+ TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
if (entry == null) {
return;
}
@@ -409,75 +425,98 @@ private static void populateEntryDetails(
traceDetails.put("length", entry.getLength());
- traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
+ if (TraceLevel.PAYLOAD == level && entry.getDataBuffer() != null) {
+ traceMetadataAndPayload(
+ "payload", entry.getDataBuffer().slice(), traceDetails, maxBinaryDataLength);
+ }
}
- public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+ public static Map<String, Object> getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) {
if (publishContext == null) {
return null;
}
Map<String, Object> details = new TreeMap<>();
- populatePublishContext(publishContext, details);
+ populatePublishContext(level, publishContext, details);
return details;
}
- private static void populatePublishContext(
+ private static void populatePublishContext(TraceLevel level,
Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
- traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
- traceDetails.put("isChunked", publishContext.isChunked());
- traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+ traceDetails.put("sequenceId", publishContext.getSequenceId());
traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
traceDetails.put("msgSize", publishContext.getMsgSize());
- if (publishContext.getOriginalProducerName() != null) {
- traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
- traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+
+ if (TraceLevel.PAYLOAD == level) {
+ traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+ traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+ traceDetails.put("isChunked", publishContext.isChunked());
+ if (publishContext.getOriginalProducerName() != null) {
+ traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+ traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+ }
}
- traceDetails.put("sequenceId", publishContext.getSequenceId());
}
- public static void traceByteBuf(
- String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
- if (buf == null || maxBinaryDataLength <= 0) return;
+ /** this will release metadataAndPayload */
+ public static void traceMetadataAndPayload(
+ String key,
+ ByteBuf metadataAndPayload,
+ Map<String, Object> traceDetails,
+ int maxPayloadLength) {
+ if (metadataAndPayload == null) return;
+ if (maxPayloadLength <= 0) {
+ metadataAndPayload.release();
+ return;
+ }
try {
+ // advance readerIndex
+ MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
+
+ // todo: do we need to trace this metadata?
+ populateMessageMetadataDetails(metadata, traceDetails);
+
+ // Decode if needed
+ CompressionCodec codec =
+ CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
+ ByteBuf uncompressedPayload =
+ codec.decode(metadataAndPayload, metadata.getUncompressedSize());
+ traceByteBuf(key, uncompressedPayload, traceDetails, maxPayloadLength);
+ } catch (Throwable t) {
+ log.error("Failed to trace metadataAndPayload", t);
+ } finally {
+ metadataAndPayload.release();
+ }
+ }
- final ByteBuf metadataAndPayload = buf.retainedDuplicate();
- ByteBuf uncompressedPayload = null;
- try {
- // advance readerIndex
- MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
-
- // todo: do we need to trace this metadata?
- populateMessageMetadataDetails(metadata, traceDetails);
-
- // Decode if needed
- CompressionCodec codec =
- CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
- uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
-
- // todo: does this require additional steps if messages are batched?
- if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) {
- String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8);
- traceDetails.put(key, dataAsString);
- } else {
- String dataAsString =
- uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8);
- traceDetails.put(key, dataAsString + "...");
- }
- } finally {
- metadataAndPayload.release();
- if (uncompressedPayload != null) {
- uncompressedPayload.release();
- }
+ /** this will release payload */
+ public static void traceByteBuf(
+ String key, ByteBuf payload, Map<String, Object> traceDetails, int maxPayloadLength) {
+ if (payload == null) return;
+
+ if (maxPayloadLength <= 0) {
+ payload.release();
+ return;
+ }
+
+ try {
+ // todo: does this require additional steps if messages are batched?
+ String dataAsString = payload.toString(StandardCharsets.UTF_8);
+ if (dataAsString.length() > maxPayloadLength + 3) {
+ dataAsString = dataAsString.substring(0, maxPayloadLength) + "...";
}
+ traceDetails.put(key, dataAsString);
} catch (Throwable t) {
log.error("Failed to convert ByteBuf to string", t);
- if (buf.readableBytes() < maxBinaryDataLength + 3) {
- traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
+ if (payload.readableBytes() < maxPayloadLength) {
+ traceDetails.put(key, "0x" + Hex.encodeHexString(payload.nioBuffer()));
} else {
- traceDetails.put(
- key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "...");
+ ByteBuf buf = payload.slice(0, maxPayloadLength / 2);
+ traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()) + "...");
+ buf.release();
}
+ } finally {
+ payload.release();
}
}
}
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 9fa3e0d4..3a87a5cf 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -20,9 +20,11 @@
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.TreeMap;
import org.junit.jupiter.api.Test;
@@ -46,21 +48,22 @@ void traceTest() {
traces.clear();
trace(mockTracer, null);
assertEquals(1, traces.size());
- assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
+ assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":null}", traces.get(0));
Map<String, Object> map = new TreeMap<>();
traces.clear();
trace(mockTracer, map);
assertEquals(1, traces.size());
- assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+ assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{}}", traces.get(0));
map.put("key1", "value1");
traces.clear();
trace(mockTracer, map);
assertEquals(1, traces.size());
- assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+ assertEquals(
+ "{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
}
// todo:
@@ -96,29 +99,40 @@ void traceTest() {
void traceByteBufTest() {
Map<String, Object> traceDetails = new TreeMap<>();
- int maxBinaryDataLength = 1024;
+ int maxBinaryDataLength = 100;
traceByteBuf("key", null, traceDetails, maxBinaryDataLength);
assertEquals(0, traceDetails.size());
+ Random rand = new Random();
+
ByteBuf small = Unpooled.buffer(20);
for (int i = 0; i < 20; i++) {
- small.writeByte(i);
+ char randomChar = (char) (rand.nextInt(26) + 'a');
+ small.writeByte(randomChar);
}
+ String smallStr = small.toString(StandardCharsets.UTF_8);
+ assertEquals(1, small.refCnt());
+
traceByteBuf("key", small, traceDetails, maxBinaryDataLength);
assertEquals(1, traceDetails.size());
- assertEquals(42, ((String) traceDetails.get("key")).length());
- assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+ assertEquals(20, ((String) traceDetails.get("key")).length());
+ assertEquals(smallStr, traceDetails.get("key"));
+ assertEquals(0, small.refCnt());
ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100);
for (int i = 0; i < maxBinaryDataLength + 100; i++) {
- big.writeByte(i);
+ char randomChar = (char) (rand.nextInt(26) + 'a');
+ big.writeByte(randomChar);
}
+ assertEquals(1, big.refCnt());
+ String bigStr = big.toString(StandardCharsets.UTF_8);
traceDetails.clear();
traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
assertEquals(1, traceDetails.size());
- assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
- assertTrue(((String) traceDetails.get("key")).startsWith("0x000102"));
+ assertEquals(maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
+ assertEquals(bigStr.substring(0, maxBinaryDataLength) + "...", traceDetails.get("key"));
+ assertEquals(0, big.refCnt());
}
}
From db0ad0c103f33eeecf12deee53706edfc864bd25 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 10 May 2024 17:51:50 -0700
Subject: [PATCH 29/29] deduped messages, aded stats etc
---
.../oss/pulsar/jms/tracing/BrokerTracing.java | 95 +++++++++----------
.../oss/pulsar/jms/tracing/TracingUtils.java | 33 +++----
2 files changed, 64 insertions(+), 64 deletions(-)
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index cfcffa90..13999322 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -367,8 +367,10 @@ private static void addMinimumProducerDetails(
if (producer.getAccessMode() != null) {
traceDetails.put("accessMode", producer.getAccessMode().name());
}
- traceDetails.put("clientHost",
- TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+ traceDetails.put(
+ "clientHost",
+ TracingUtils.hostNameOf(
+ producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
if (producer.getTopic() != null) {
traceDetails.put(
@@ -389,8 +391,10 @@ private static void addMinimumConsumerSubscriptionDetails(
if (consumer != null) {
traceDetails.put("consumerName", consumer.consumerName());
traceDetails.put("consumerId", consumer.consumerId());
- traceDetails.put("clientHost",
- TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
+ traceDetails.put(
+ "clientHost",
+ TracingUtils.hostNameOf(
+ consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
traceDetails.put("authRole", consumer.cnx().getAuthRole());
}
@@ -451,13 +455,16 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
traceDetails.put("metadata", metadata);
traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
+ Map<String, Object> statsTrace = new TreeMap<>();
PublisherStatsImpl stats = producer.getStats();
- traceDetails.put("connectedSince", stats.getConnectedSince());
- traceDetails.put("closedAt", DateFormatter.now());
- traceDetails.put("averageMsgSize", stats.getAverageMsgSize());
- traceDetails.put("msgRateIn", stats.getMsgRateIn());
- traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
+
+ statsTrace.put("connectedSince", stats.getConnectedSince());
+ statsTrace.put("closedAt", DateFormatter.now());
+ statsTrace.put("averageMsgSize", stats.getAverageMsgSize());
+ statsTrace.put("msgRateIn", stats.getMsgRateIn());
+ statsTrace.put("msgThroughputIn", stats.getMsgThroughputIn());
// no message count in stats? stats.getCount() is not it
+ traceDetails.put("stats", statsTrace);
trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails);
}
@@ -490,16 +497,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
ConsumerStatsImpl stats = consumer.getStats();
- traceDetails.put("connectedSince", stats.getConnectedSince());
- traceDetails.put("closedAt", DateFormatter.now());
- traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
- traceDetails.put("msgRateOut", stats.getMsgRateOut());
- traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut());
- traceDetails.put("msgOutCounter", stats.getMsgOutCounter());
- traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
- traceDetails.put("unackedMessages", stats.getUnackedMessages());
- traceDetails.put("messageAckRate", stats.getMessageAckRate());
- traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
+ Map<String, Object> statsTrace = new TreeMap<>();
+ statsTrace.put("connectedSince", stats.getConnectedSince());
+ statsTrace.put("closedAt", DateFormatter.now());
+ statsTrace.put("averageMsgSize", stats.getAvgMessagesPerEntry());
+ statsTrace.put("msgRateOut", stats.getMsgRateOut());
+ statsTrace.put("msgThroughputOut", stats.getMsgThroughputOut());
+ statsTrace.put("msgOutCounter", stats.getMsgOutCounter());
+ statsTrace.put("bytesOutCounter", stats.getBytesOutCounter());
+ statsTrace.put("unackedMessages", stats.getUnackedMessages());
+ statsTrace.put("messageAckRate", stats.getMessageAckRate());
+ statsTrace.put("msgRateRedeliver", stats.getMsgRateRedeliver());
+ statsTrace.put("readPositionWhenJoining", stats.getReadPositionWhenJoining());
+ Subscription sub = consumer.getSubscription();
+ if (sub != null) {
+ statsTrace.put("subscriptionApproxBacklog", sub.getNumberOfEntriesInBacklog(false));
+ statsTrace.put("subscriptionMsgRateExpired", sub.getExpiredMessageRate());
+ }
+ traceDetails.put("stats", statsTrace);
trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails);
}
@@ -524,11 +539,11 @@ public void onMessagePublish(
if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
traceMetadataAndPayload(
- "headersAndPayload",
- headersAndPayload.slice(),
+ "payload",
+ headersAndPayload.retainedDuplicate(),
headersAndPayloadDetails,
maxPayloadLength);
- traceDetails.put("payload", headersAndPayloadDetails);
+ traceDetails.put("headersAndPayload", headersAndPayloadDetails);
}
trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
@@ -569,7 +584,9 @@ public void beforeSendMessage(
addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
- traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength));
+ traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
+
+ traceDetails.put("headersAndPayload", getEntryDetails(level, entry, maxPayloadLength));
trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
}
@@ -588,11 +605,11 @@ public void messageDispatched(
if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
traceMetadataAndPayload(
- "headersAndPayload",
- headersAndPayload.slice(),
+ "payload",
+ headersAndPayload.retainedDuplicate(),
headersAndPayloadDetails,
maxPayloadLength);
- traceDetails.put("payload", headersAndPayloadDetails);
+ traceDetails.put("headersAndPayload", headersAndPayloadDetails);
}
trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
@@ -604,35 +621,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
TraceLevel level = getTracingLevel(consumer);
if (consumer != null && level == TraceLevel.OFF) return;
- EventSubCategory subcategory = EventSubCategory.ACKED;
-
Map<String, Object> traceDetails = new TreeMap<>();
addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
- if (consumer == null) {
- // ack with empty consumer == message filtered by JMSFilter
- traceDetails.put("reason", "filtered by JMSFilter");
- subcategory = EventSubCategory.FILTERED;
- } else {
- // todo: am I right that unacked/nacked messages never go through broker interceptor?
- // in this case we need consumer interceptor to track nacks
- traceDetails.put("reason", "acked");
- }
-
- if (consumer != null && consumer.getSubscription() != null) {
- Subscription sub = consumer.getSubscription();
- traceDetails.put("subscriptionName", sub.getName());
- traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- traceDetails.put("subscriptionType", sub.getType().name());
- }
Map<String, Object> ackDetails = new TreeMap<>();
if (ackCmd.hasAckType()) {
ackDetails.put("type", ackCmd.getAckType().name());
}
- if (ackCmd.hasConsumerId()) {
- ackDetails.put("ackConsumerId", ackCmd.getConsumerId());
- }
ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
ackDetails.put(
"messageIds",
@@ -652,6 +648,9 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
traceDetails.put("ack", ackDetails);
+ EventSubCategory subcategory =
+ consumer == null ? EventSubCategory.FILTERED : EventSubCategory.ACKED;
+
trace(EventCategory.MSG, subcategory, traceDetails);
}
@@ -708,7 +707,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
Map<String, Object> traceDetails = new TreeMap<>();
- traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
+ traceDetails.put("host", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
traceDetails.put("protocol", request.getProtocol());
traceDetails.put("scheme", request.getScheme());
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 4250e867..c6e6534b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -139,8 +139,10 @@ public String load(String clientAddress) {
});
public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) {
- if (clientAddress == null || clientAddress.isEmpty()
- || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) {
+ if (clientAddress == null
+ || clientAddress.isEmpty()
+ || clientSourceAddressAndPort == null
+ || !clientSourceAddressAndPort.contains(":")) {
return "unknown/null";
}
@@ -207,7 +209,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
if (cnx == null) {
return;
}
- traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
+ traceDetails.put(
+ "clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
traceDetails.put("authRole", cnx.getAuthRole());
traceDetails.put("clientVersion", cnx.getClientVersion());
traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
@@ -300,15 +303,12 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
traceDetails.put("name", consumer.consumerName());
traceDetails.put("consumerId", consumer.consumerId());
- Subscription sub = consumer.getSubscription();
- if (sub != null) {
- traceDetails.put("subscriptionName", sub.getName());
- traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
- }
traceDetails.put("priorityLevel", consumer.getPriorityLevel());
traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
- traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
+ traceDetails.put(
+ "clientHost",
+ hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
traceDetails.put("metadata", consumer.getMetadata());
traceDetails.put("unackedMessages", consumer.getUnackedMessages());
@@ -340,7 +340,9 @@ private static void populateProducerDetails(
"topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
}
- traceDetails.put("clientHost", hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+ traceDetails.put(
+ "clientHost",
+ hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
traceDetails.put("metadata", producer.getMetadata());
@@ -421,17 +423,16 @@ private static void populateEntryDetails(
return;
}
- traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
-
traceDetails.put("length", entry.getLength());
if (TraceLevel.PAYLOAD == level && entry.getDataBuffer() != null) {
traceMetadataAndPayload(
- "payload", entry.getDataBuffer().slice(), traceDetails, maxBinaryDataLength);
+ "payload", entry.getDataBuffer().retainedDuplicate(), traceDetails, maxBinaryDataLength);
}
}
- public static Map<String, Object> getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) {
+ public static Map<String, Object> getPublishContextDetails(
+ TraceLevel level, Topic.PublishContext publishContext) {
if (publishContext == null) {
return null;
}
@@ -441,8 +442,8 @@ public static Map<String, Object> getPublishContextDetails(TraceLevel level, Top
return details;
}
- private static void populatePublishContext(TraceLevel level,
- Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+ private static void populatePublishContext(
+ TraceLevel level, Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
traceDetails.put("sequenceId", publishContext.getSequenceId());
traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
traceDetails.put("msgSize", publishContext.getMsgSize());