From 59c2422478856aaee92c3ecc7b9434f082e8483f Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 11 Apr 2024 17:38:28 -0700
Subject: [PATCH 01/29] poc, take 1

---
 pom.xml                                       |   1 +
 pulsar-tracing/pom.xml                        |  82 +++
 .../oss/pulsar/tracing/BrokerTracing.java     | 645 ++++++++++++++++++
 .../META-INF/services/broker_interceptor.yml  |   3 +
 pulsar-tracing/src/test/resources/log4j2.xml  |  41 ++
 5 files changed, 772 insertions(+)
 create mode 100644 pulsar-tracing/pom.xml
 create mode 100644 pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
 create mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
 create mode 100644 pulsar-tracing/src/test/resources/log4j2.xml

diff --git a/pom.xml b/pom.xml
index 0e83aa9a..8f9c2802 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,7 @@
   <modules>
     <module>activemq-filters</module>
     <module>pulsar-jms-filters</module>
+    <module>pulsar-tracing</module>
     <module>pulsar-jms-admin-api</module>
     <module>pulsar-jms</module>
     <module>resource-adapter</module>
diff --git a/pulsar-tracing/pom.xml b/pulsar-tracing/pom.xml
new file mode 100644
index 00000000..e4a65161
--- /dev/null
+++ b/pulsar-tracing/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright DataStax, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>pulsar-jms-parent</artifactId>
+    <groupId>com.datastax.oss</groupId>
+    <version>4.1.1-alpha-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>pulsar-tracing</artifactId>
+  <packaging>jar</packaging>
+  <name>DataStax Starlight for JMS - Broker Side Filters</name>
+  <properties>
+    <build.dir>${project.build.directory}</build.dir>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.github.spotbugs</groupId>
+      <artifactId>spotbugs-annotations</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>${pulsar.groupId}</groupId>
+      <artifactId>pulsar-broker</artifactId>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.nifi</groupId>
+        <artifactId>nifi-nar-maven-plugin</artifactId>
+        <version>1.3.2</version>
+        <extensions>true</extensions>
+        <configuration>
+          <finalName>pulsar-tracing-${project.version}</finalName>
+          <classifier>nar</classifier>
+        </configuration>
+        <executions>
+          <execution>
+            <id>default-nar</id>
+            <phase>package</phase>
+            <goals>
+              <goal>nar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
new file mode 100644
index 00000000..e18e2cec
--- /dev/null
+++ b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
@@ -0,0 +1,645 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastax.oss.pulsar.tracing;
+
+import io.netty.buffer.ByteBuf;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.intercept.BrokerInterceptor;
+import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.broker.service.Producer;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.Subscription;
+import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.BaseCommand;
+import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.naming.TopicName;
+
+@Slf4j
+public class BrokerTracing implements BrokerInterceptor {
+
+  static final int MAX_DATA_LENGTH = 1024;
+
+  public enum EventReasons {
+    ADMINISTRATIVE,
+    MESSAGE,
+    TRANSACTION,
+    SERVLET,
+  }
+
+  public enum TraceLevel {
+    NONE,
+    MIMIMAL,
+    BASIC,
+    FULL
+  }
+
+  //  private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
+  private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+
+  public void initialize(PulsarService pulsarService) {
+    //    defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
+    //    defaultEnabledEvents.add(EventReasons.MESSAGE);
+    //    defaultEnabledEvents.add(EventReasons.TRANSACTION);
+  }
+
+  @Override
+  public void close() {}
+
+  private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
+    return getEnabledEvents(cnx.getBrokerService().getPulsar());
+  }
+
+  private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
+    // todo: do this less frequently
+
+    String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", "");
+
+    Set<EventReasons> enabledEvents = new HashSet<>();
+
+    for (String event : events.split(",")) {
+      try {
+        enabledEvents.add(EventReasons.valueOf(event.trim()));
+      } catch (IllegalArgumentException e) {
+        log.warn("Invalid event: {}. Skipping", event);
+      }
+    }
+
+    return enabledEvents;
+  }
+
+  private static TraceLevel getTracingLevel(ServerCnx cnx) {
+    // todo: do this less frequently
+    String level =
+        cnx.getBrokerService()
+            .getPulsar()
+            .getConfiguration()
+            .getProperties()
+            .getProperty("tracing_level", defaultTraceLevel.toString());
+    try {
+      return TraceLevel.valueOf(level);
+    } catch (IllegalArgumentException e) {
+      log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+      return defaultTraceLevel;
+    }
+  }
+
+  private static void populateConnectionDetails(
+      TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+    if (cnx == null) {
+      traceDetails.put("ServerCnx", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
+        break;
+      case BASIC:
+        populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+
+        traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
+        traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+        break;
+      case FULL:
+        populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+        populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
+
+        traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
+        traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod());
+        traceDetails.put(
+            "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populateSubscriptionDetails(
+      TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+    if (sub == null) {
+      traceDetails.put("Subscription", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("Subscription.name", sub.getName());
+        traceDetails.put(
+            "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+        traceDetails.put("Subscription.type", sub.getType().name());
+        break;
+      case BASIC:
+        populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+
+        if (sub.getConsumers() != null) {
+          traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
+          traceDetails.put(
+              "Subscription.namesOfConsumers",
+              sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
+        }
+
+        break;
+      case FULL:
+        populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+        populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
+
+        traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populateConsumerDetails(
+      TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+    if (consumer == null) {
+      traceDetails.put("Consumer", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("Consumer.name", consumer.consumerName());
+        traceDetails.put("Consumer.consumerId", consumer.consumerId());
+        if (consumer.getSubscription() != null) {
+          traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName());
+          traceDetails.put(
+              "Consumer.topicName",
+              TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+        }
+        break;
+      case BASIC:
+        populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+
+        traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
+        traceDetails.put("Consumer.subType", consumer.subType().name());
+        traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
+        break;
+      case FULL:
+        populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+        populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+
+        traceDetails.put("Consumer.metadata", consumer.getMetadata());
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populateProducerDetails(
+      TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+    if (producer == null) {
+      traceDetails.put("Producer", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("Producer.producerId", producer.getProducerId());
+        traceDetails.put("Producer.producerName", producer.getProducerName());
+        traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
+        if (producer.getTopic() != null) {
+          traceDetails.put(
+              "Producer.topicName",
+              TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+        }
+        break;
+      case BASIC:
+        populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+
+        traceDetails.put("Producer.clientAddress", producer.getClientAddress());
+        break;
+      case FULL:
+        populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+
+        traceDetails.put("Producer.metadata", producer.getMetadata());
+        if (producer.getSchemaVersion() != null) {
+          traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString());
+        }
+        traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populateMessageMetadataDetails(
+      TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+    if (msgMetadata == null) {
+      traceDetails.put("MessageMetadata", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
+        traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
+        traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
+        break;
+      case BASIC:
+        populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+
+        traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
+        traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
+        traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+        break;
+      case FULL:
+        populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+        populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
+
+        traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
+        traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime());
+        traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom());
+        traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populateEntryDetails(
+      TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+    if (entry == null) {
+      traceDetails.put("Entry", "null");
+      return;
+    }
+
+    switch (level) {
+      case MIMIMAL:
+        traceDetails.put("Entry.ledgerId", entry.getLedgerId());
+        traceDetails.put("Entry.entryId", entry.getEntryId());
+        break;
+      case BASIC:
+        populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+
+        traceDetails.put("Entry.length", entry.getLength());
+        break;
+      case FULL:
+        populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+
+        traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
+        break;
+      default:
+        log.debug("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  private static void populatePublishContext(
+      Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+    traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage());
+    traceDetails.put("publishContext.isChunked", publishContext.isChunked());
+    traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages());
+
+    traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp());
+    traceDetails.put("publishContext.msgSize", publishContext.getMsgSize());
+    traceDetails.put("publishContext.producerName", publishContext.getProducerName());
+    traceDetails.put(
+        "publishContext.originalProducerName", publishContext.getOriginalProducerName());
+    traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId());
+    traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId());
+  }
+
+  private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
+    if (buf == null) return;
+
+    if (buf.readableBytes() < MAX_DATA_LENGTH) {
+      traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+    } else {
+      traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+    }
+  }
+
+  private static TraceLevel getTracingLevel(Subscription sub) {
+    if (sub == null
+        || sub.getSubscriptionProperties() == null
+        || !sub.getSubscriptionProperties().containsKey("trace")) {
+      return TraceLevel.NONE;
+    }
+    try {
+      return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+    } catch (IllegalArgumentException e) {
+      log.debug(
+          "Invalid tracing level: {}. Setting to NONE for subscription {}",
+          sub.getSubscriptionProperties().get("trace"),
+          sub);
+      return TraceLevel.NONE;
+    }
+  }
+
+  private static TraceLevel getTracingLevel(Producer producer) {
+    if (producer == null
+        || producer.getMetadata() == null
+        || !producer.getMetadata().containsKey("trace")) {
+      return TraceLevel.NONE;
+    }
+    try {
+      return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+    } catch (IllegalArgumentException e) {
+      log.debug(
+          "Invalid tracing level: {}. Setting to NONE for producer {}",
+          producer.getMetadata().get("trace"),
+          producer);
+      return TraceLevel.NONE;
+    }
+  }
+
+  //    private boolean needToTraceTopic(Topic topic) {
+  //        if (topic == null) return false;
+  //
+  //        // todo: how to read topic props
+  //        return true;
+  //    }
+
+  /* ***************************
+   **  Administrative events
+   ******************************/
+
+  public void onConnectionCreated(ServerCnx cnx) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    log.info("Connection created: {}", traceDetails);
+  }
+
+  public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateProducerDetails(level, producer, traceDetails);
+
+    traceDetails.put("metadata", metadata);
+
+    log.info("Producer created: {}", traceDetails);
+  }
+
+  public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateProducerDetails(level, producer, traceDetails);
+
+    traceDetails.put("metadata", metadata);
+
+    log.info("Producer closed: {}", traceDetails);
+  }
+
+  public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateConsumerDetails(level, consumer, traceDetails);
+    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+    traceDetails.put("metadata", metadata);
+
+    log.info("Consumer created: {}", traceDetails);
+  }
+
+  public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateConsumerDetails(level, consumer, traceDetails);
+    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+    traceDetails.put("metadata", metadata);
+
+    log.info("Consumer closed: {}", traceDetails);
+  }
+
+  public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    traceDetails.put("command", command.toString());
+    log.info("Pulsar command called: {}", traceDetails);
+  }
+
+  public void onConnectionClosed(ServerCnx cnx) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+
+    TraceLevel level = getTracingLevel(cnx);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    log.info("Connection closed: {}", traceDetails);
+  }
+
+  /* ***************************
+   **  Message events
+   ******************************/
+
+  public void beforeSendMessage(
+      Subscription subscription,
+      Entry entry,
+      long[] ackSet,
+      MessageMetadata msgMetadata,
+      Consumer consumer) {
+    if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar())
+        .contains(EventReasons.MESSAGE)) return;
+
+    TraceLevel level = getTracingLevel(subscription);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConsumerDetails(level, consumer, traceDetails);
+    populateSubscriptionDetails(level, subscription, traceDetails);
+    populateEntryDetails(level, entry, traceDetails);
+    populateMessageMetadataDetails(level, msgMetadata, traceDetails);
+
+    log.info("Before sending message: {}", traceDetails);
+  }
+
+  public void onMessagePublish(
+      Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
+
+    if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar())
+        .contains(EventReasons.MESSAGE)) return;
+
+    TraceLevel level = getTracingLevel(producer);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateProducerDetails(level, producer, traceDetails);
+
+    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+
+    populatePublishContext(publishContext, traceDetails);
+
+    log.info("Message publish: {}", traceDetails);
+  }
+
+  public void messageProduced(
+      ServerCnx cnx,
+      Producer producer,
+      long startTimeNs,
+      long ledgerId,
+      long entryId,
+      Topic.PublishContext publishContext) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+    TraceLevel level = getTracingLevel(producer);
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateProducerDetails(level, producer, traceDetails);
+    populatePublishContext(publishContext, traceDetails);
+
+    traceDetails.put("ledgerId", ledgerId);
+    traceDetails.put("entryId", entryId);
+    traceDetails.put("startTimeNs", startTimeNs);
+
+    log.info("Message produced: {}", traceDetails);
+  }
+
+  public void messageDispatched(
+      ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+    TraceLevel level = getTracingLevel(consumer.getSubscription());
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateConsumerDetails(level, consumer, traceDetails);
+    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+    traceDetails.put("ledgerId", ledgerId);
+    traceDetails.put("entryId", entryId);
+
+    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+
+    log.info("After dispatching message: {}", traceDetails);
+  }
+
+  public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
+    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+
+    TraceLevel level = getTracingLevel(consumer.getSubscription());
+    if (level == TraceLevel.NONE) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    populateConnectionDetails(level, cnx, traceDetails);
+    populateConsumerDetails(level, consumer, traceDetails);
+    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
+
+    traceDetails.put("ack.type", ackCmd.getAckType().name());
+    traceDetails.put("ack.consumerId", ackCmd.getConsumerId());
+    traceDetails.put(
+        "ack.messageIds",
+        ackCmd
+            .getMessageIdsList()
+            .stream()
+            .map(x -> x.getLedgerId() + ":" + x.getEntryId())
+            .collect(Collectors.toList()));
+
+    log.info("Message acked: {}", traceDetails);
+  }
+
+  /* ***************************
+   **  Transaction events
+   ******************************/
+
+  public void txnOpened(long tcId, String txnID) {
+    //    if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+    Map<String, Object> traceDetails = new TreeMap<>();
+    traceDetails.put("tcId", tcId);
+    traceDetails.put("txnID", txnID);
+
+    log.info("Transaction opened: {}", traceDetails);
+    //    }
+  }
+
+  public void txnEnded(String txnID, long txnAction) {
+    //    if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+    Map<String, Object> traceDetails = new TreeMap<>();
+    traceDetails.put("txnID", txnID);
+    traceDetails.put("txnAction", txnAction);
+
+    log.info("Transaction closed: {}", traceDetails);
+    //    }
+  }
+
+  /* ***************************
+   **  Servlet events
+   ******************************/
+
+  public void onWebserviceRequest(ServletRequest request)
+      throws IOException, ServletException, InterceptException {
+    //    if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
+    //      log.info("onWebserviceRequest: Tracing servlet requests not supported");
+    //    }
+  }
+
+  public void onWebserviceResponse(ServletRequest request, ServletResponse response)
+      throws IOException, ServletException {
+    //    if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
+    //      log.info("onWebserviceResponse: Tracing servlet requests not supported");
+    //    }
+  }
+
+  // not needed
+  // public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+}
diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
new file mode 100644
index 00000000..38748bb9
--- /dev/null
+++ b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
@@ -0,0 +1,3 @@
+interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing
+name: pulsar-tracing
+description: Starlight for JMS - support for server side tracing
\ No newline at end of file
diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-tracing/src/test/resources/log4j2.xml
new file mode 100644
index 00000000..485f6705
--- /dev/null
+++ b/pulsar-tracing/src/test/resources/log4j2.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright DataStax, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<Configuration status="INFO">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="DEBUG">
+      <AppenderRef ref="Console"/>
+    </Root>
+    <logger name="org.apache.bookkeeper" level="error" additivity="true"/>
+    <logger name="org.apache.bookkeeper.mledger" level="info" additivity="true"/>
+    <logger name="org.apache.pulsar" level="info" additivity="true"/>
+    <logger name="org.apache.zookeeper" level="error" additivity="true"/>
+    <logger name="org.apache.curator" level="error" additivity="true"/>
+    <logger name="org.eclipse.jetty" level="error" additivity="true"/>
+    <logger name="org.glassfish.jersey" level="error" additivity="false"/>
+    <logger name="com.datastax.oss.pulsar.jms" level="debug" additivity="true"/>
+    <logger name="com.datastax.oss.pulsar.jms.utils" level="info" additivity="false"/>
+    <logger name="io.netty" level="error" additivity="true"/>
+    <logger name="org.asynchttpclient" level="error" additivity="true"/>
+  </Loggers>
+</Configuration>

From 91e51864fd52f409366310923ada743e71833bdd Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 11:12:41 -0700
Subject: [PATCH 02/29] various renamings and tweaks

---
 pom.xml                                       |  2 +-
 .../pom.xml                                   |  4 +-
 .../pulsar/jms}/tracing/BrokerTracing.java    | 91 ++++++++++---------
 .../META-INF/services/broker_interceptor.yml  |  3 +
 .../src/test/resources/log4j2.xml             |  0
 .../META-INF/services/broker_interceptor.yml  |  3 -
 6 files changed, 53 insertions(+), 50 deletions(-)
 rename {pulsar-tracing => pulsar-jms-tracing}/pom.xml (95%)
 rename {pulsar-tracing/src/main/java/com/datastax/oss/pulsar => pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms}/tracing/BrokerTracing.java (89%)
 create mode 100644 pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
 rename {pulsar-tracing => pulsar-jms-tracing}/src/test/resources/log4j2.xml (100%)
 delete mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml

diff --git a/pom.xml b/pom.xml
index 8f9c2802..a24e298a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@
   <modules>
     <module>activemq-filters</module>
     <module>pulsar-jms-filters</module>
-    <module>pulsar-tracing</module>
+    <module>pulsar-jms-tracing</module>
     <module>pulsar-jms-admin-api</module>
     <module>pulsar-jms</module>
     <module>resource-adapter</module>
diff --git a/pulsar-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
similarity index 95%
rename from pulsar-tracing/pom.xml
rename to pulsar-jms-tracing/pom.xml
index e4a65161..2d4536d8 100644
--- a/pulsar-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -23,7 +23,7 @@
     <version>4.1.1-alpha-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
-  <artifactId>pulsar-tracing</artifactId>
+  <artifactId>pulsar-jms-tracing</artifactId>
   <packaging>jar</packaging>
   <name>DataStax Starlight for JMS - Broker Side Filters</name>
   <properties>
@@ -64,7 +64,7 @@
         <version>1.3.2</version>
         <extensions>true</extensions>
         <configuration>
-          <finalName>pulsar-tracing-${project.version}</finalName>
+          <finalName>pulsar-jms-tracing-${project.version}</finalName>
           <classifier>nar</classifier>
         </configuration>
         <executions>
diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
similarity index 89%
rename from pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
rename to pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index e18e2cec..82e6ec3b 100644
--- a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.datastax.oss.pulsar.tracing;
+package com.datastax.oss.pulsar.jms.tracing;
 
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
@@ -44,6 +44,8 @@
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
 
+  public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
   static final int MAX_DATA_LENGTH = 1024;
 
   public enum EventReasons {
@@ -55,7 +57,7 @@ public enum EventReasons {
 
   public enum TraceLevel {
     NONE,
-    MIMIMAL,
+    MINIMAL,
     BASIC,
     FULL
   }
@@ -79,7 +81,8 @@ private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
   private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
     // todo: do this less frequently
 
-    String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", "");
+    String events =
+        pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
 
     Set<EventReasons> enabledEvents = new HashSet<>();
 
@@ -101,7 +104,7 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
             .getPulsar()
             .getConfiguration()
             .getProperties()
-            .getProperty("tracing_level", defaultTraceLevel.toString());
+            .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
     try {
       return TraceLevel.valueOf(level);
     } catch (IllegalArgumentException e) {
@@ -118,17 +121,17 @@ private static void populateConnectionDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
         break;
       case BASIC:
-        populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
 
         traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
         traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
         break;
       case FULL:
-        populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails);
+        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
         populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
 
         traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
@@ -137,7 +140,7 @@ private static void populateConnectionDetails(
             "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -150,14 +153,14 @@ private static void populateSubscriptionDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("Subscription.name", sub.getName());
         traceDetails.put(
             "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
         traceDetails.put("Subscription.type", sub.getType().name());
         break;
       case BASIC:
-        populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
 
         if (sub.getConsumers() != null) {
           traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
@@ -168,13 +171,13 @@ private static void populateSubscriptionDetails(
 
         break;
       case FULL:
-        populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails);
+        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
         populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
 
         traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -187,7 +190,7 @@ private static void populateConsumerDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("Consumer.name", consumer.consumerName());
         traceDetails.put("Consumer.consumerId", consumer.consumerId());
         if (consumer.getSubscription() != null) {
@@ -198,20 +201,20 @@ private static void populateConsumerDetails(
         }
         break;
       case BASIC:
-        populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
 
         traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
         traceDetails.put("Consumer.subType", consumer.subType().name());
         traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
         break;
       case FULL:
-        populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails);
+        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
         populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
 
         traceDetails.put("Consumer.metadata", consumer.getMetadata());
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -224,7 +227,7 @@ private static void populateProducerDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("Producer.producerId", producer.getProducerId());
         traceDetails.put("Producer.producerName", producer.getProducerName());
         traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
@@ -235,12 +238,12 @@ private static void populateProducerDetails(
         }
         break;
       case BASIC:
-        populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
 
         traceDetails.put("Producer.clientAddress", producer.getClientAddress());
         break;
       case FULL:
-        populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails);
+        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
         populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
 
         traceDetails.put("Producer.metadata", producer.getMetadata());
@@ -250,7 +253,7 @@ private static void populateProducerDetails(
         traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -263,20 +266,20 @@ private static void populateMessageMetadataDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
         traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
         traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
         break;
       case BASIC:
-        populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
 
         traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
         traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
         traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
         break;
       case FULL:
-        populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails);
+        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
         populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
 
         traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
@@ -285,7 +288,7 @@ private static void populateMessageMetadataDetails(
         traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -298,23 +301,23 @@ private static void populateEntryDetails(
     }
 
     switch (level) {
-      case MIMIMAL:
+      case MINIMAL:
         traceDetails.put("Entry.ledgerId", entry.getLedgerId());
         traceDetails.put("Entry.entryId", entry.getEntryId());
         break;
       case BASIC:
-        populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
 
         traceDetails.put("Entry.length", entry.getLength());
         break;
       case FULL:
-        populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails);
+        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
         populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
 
         traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
         break;
       default:
-        log.debug("Unknown tracing level: {}", level);
+        log.warn("Unknown tracing level: {}", level);
         break;
     }
   }
@@ -353,7 +356,7 @@ private static TraceLevel getTracingLevel(Subscription sub) {
     try {
       return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
     } catch (IllegalArgumentException e) {
-      log.debug(
+      log.warn(
           "Invalid tracing level: {}. Setting to NONE for subscription {}",
           sub.getSubscriptionProperties().get("trace"),
           sub);
@@ -370,7 +373,7 @@ private static TraceLevel getTracingLevel(Producer producer) {
     try {
       return TraceLevel.valueOf(producer.getMetadata().get("trace"));
     } catch (IllegalArgumentException e) {
-      log.debug(
+      log.warn(
           "Invalid tracing level: {}. Setting to NONE for producer {}",
           producer.getMetadata().get("trace"),
           producer);
@@ -397,7 +400,7 @@ public void onConnectionCreated(ServerCnx cnx) {
 
     Map<String, Object> traceDetails = new TreeMap<>();
     populateConnectionDetails(level, cnx, traceDetails);
-    log.info("Connection created: {}", traceDetails);
+    tracer.info("Connection created: {}", traceDetails);
   }
 
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -412,7 +415,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
 
     traceDetails.put("metadata", metadata);
 
-    log.info("Producer created: {}", traceDetails);
+    tracer.info("Producer created: {}", traceDetails);
   }
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -427,7 +430,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
 
     traceDetails.put("metadata", metadata);
 
-    log.info("Producer closed: {}", traceDetails);
+    tracer.info("Producer closed: {}", traceDetails);
   }
 
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -443,7 +446,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
 
     traceDetails.put("metadata", metadata);
 
-    log.info("Consumer created: {}", traceDetails);
+    tracer.info("Consumer created: {}", traceDetails);
   }
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -459,7 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
 
     traceDetails.put("metadata", metadata);
 
-    log.info("Consumer closed: {}", traceDetails);
+    tracer.info("Consumer closed: {}", traceDetails);
   }
 
   public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
@@ -471,7 +474,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
     Map<String, Object> traceDetails = new TreeMap<>();
     populateConnectionDetails(level, cnx, traceDetails);
     traceDetails.put("command", command.toString());
-    log.info("Pulsar command called: {}", traceDetails);
+    tracer.info("Pulsar command called: {}", traceDetails);
   }
 
   public void onConnectionClosed(ServerCnx cnx) {
@@ -482,7 +485,7 @@ public void onConnectionClosed(ServerCnx cnx) {
 
     Map<String, Object> traceDetails = new TreeMap<>();
     populateConnectionDetails(level, cnx, traceDetails);
-    log.info("Connection closed: {}", traceDetails);
+    tracer.info("Connection closed: {}", traceDetails);
   }
 
   /* ***************************
@@ -507,7 +510,7 @@ public void beforeSendMessage(
     populateEntryDetails(level, entry, traceDetails);
     populateMessageMetadataDetails(level, msgMetadata, traceDetails);
 
-    log.info("Before sending message: {}", traceDetails);
+    tracer.info("Before sending message: {}", traceDetails);
   }
 
   public void onMessagePublish(
@@ -526,7 +529,7 @@ public void onMessagePublish(
 
     populatePublishContext(publishContext, traceDetails);
 
-    log.info("Message publish: {}", traceDetails);
+    tracer.info("Message publish: {}", traceDetails);
   }
 
   public void messageProduced(
@@ -551,7 +554,7 @@ public void messageProduced(
     traceDetails.put("entryId", entryId);
     traceDetails.put("startTimeNs", startTimeNs);
 
-    log.info("Message produced: {}", traceDetails);
+    tracer.info("Message produced: {}", traceDetails);
   }
 
   public void messageDispatched(
@@ -571,7 +574,7 @@ public void messageDispatched(
 
     traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
 
-    log.info("After dispatching message: {}", traceDetails);
+    tracer.info("After dispatching message: {}", traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -595,7 +598,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
             .map(x -> x.getLedgerId() + ":" + x.getEntryId())
             .collect(Collectors.toList()));
 
-    log.info("Message acked: {}", traceDetails);
+    tracer.info("Message acked: {}", traceDetails);
   }
 
   /* ***************************
@@ -608,7 +611,7 @@ public void txnOpened(long tcId, String txnID) {
     traceDetails.put("tcId", tcId);
     traceDetails.put("txnID", txnID);
 
-    log.info("Transaction opened: {}", traceDetails);
+    tracer.info("Transaction opened: {}", traceDetails);
     //    }
   }
 
@@ -618,7 +621,7 @@ public void txnEnded(String txnID, long txnAction) {
     traceDetails.put("txnID", txnID);
     traceDetails.put("txnAction", txnAction);
 
-    log.info("Transaction closed: {}", traceDetails);
+    tracer.info("Transaction closed: {}", traceDetails);
     //    }
   }
 
diff --git a/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
new file mode 100644
index 00000000..a47fa05f
--- /dev/null
+++ b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
@@ -0,0 +1,3 @@
+interceptorClass: com.datastax.oss.pulsar.jms.tracing.BrokerTracing
+name: jms-tracing
+description: Starlight for JMS - support for server side tracing
\ No newline at end of file
diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-jms-tracing/src/test/resources/log4j2.xml
similarity index 100%
rename from pulsar-tracing/src/test/resources/log4j2.xml
rename to pulsar-jms-tracing/src/test/resources/log4j2.xml
diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
deleted file mode 100644
index 38748bb9..00000000
--- a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing
-name: pulsar-tracing
-description: Starlight for JMS - support for server side tracing
\ No newline at end of file

From ea914204b8653be053ae27499e33f6e022627211 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 12:16:28 -0700
Subject: [PATCH 03/29] traces as json

---
 pulsar-jms-tracing/pom.xml                    |   4 +
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 315 +++++++++++-------
 2 files changed, 200 insertions(+), 119 deletions(-)

diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index 2d4536d8..c60c0b4a 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -30,6 +30,10 @@
     <build.dir>${project.build.directory}</build.dir>
   </properties>
   <dependencies>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 82e6ec3b..dd41a4c4 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,6 +15,9 @@
  */
 package com.datastax.oss.pulsar.jms.tracing;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
 import java.util.HashSet;
@@ -44,7 +47,12 @@
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
 
-  public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+  private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+  private static final ObjectMapper mapper =
+      new ObjectMapper()
+          .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
+          .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
 
   static final int MAX_DATA_LENGTH = 1024;
 
@@ -65,6 +73,19 @@ public enum TraceLevel {
   //  private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
   private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
 
+  public static void trace(String message, Map<String, Object> traceDetails) {
+    Map<String, Object> trace = new TreeMap<>();
+    trace.put("message", message);
+    trace.put("details", traceDetails);
+
+    try {
+      String loggableJsonString = mapper.writeValueAsString(trace);
+      tracer.info(loggableJsonString);
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   public void initialize(PulsarService pulsarService) {
     //    defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
     //    defaultEnabledEvents.add(EventReasons.MESSAGE);
@@ -113,31 +134,39 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
     }
   }
 
+  public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+    if (cnx == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateConnectionDetails(level, cnx, details);
+    return details;
+  }
+
   private static void populateConnectionDetails(
       TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
     if (cnx == null) {
-      traceDetails.put("ServerCnx", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString());
+        traceDetails.put("clientAddress", cnx.clientAddress().toString());
         break;
       case BASIC:
         populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
 
-        traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion());
-        traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+        traceDetails.put("clientVersion", cnx.getClientVersion());
+        traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
         break;
       case FULL:
         populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
         populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
 
-        traceDetails.put("ServerCnx.authRole", cnx.getAuthRole());
-        traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod());
-        traceDetails.put(
-            "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+        traceDetails.put("authRole", cnx.getAuthRole());
+        traceDetails.put("authMethod", cnx.getAuthMethod());
+        traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -145,27 +174,35 @@ private static void populateConnectionDetails(
     }
   }
 
+  public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+    if (sub == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateSubscriptionDetails(level, sub, details);
+    return details;
+  }
+
   private static void populateSubscriptionDetails(
       TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
     if (sub == null) {
-      traceDetails.put("Subscription", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("Subscription.name", sub.getName());
-        traceDetails.put(
-            "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-        traceDetails.put("Subscription.type", sub.getType().name());
+        traceDetails.put("name", sub.getName());
+        traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+        traceDetails.put("type", sub.getType().name());
         break;
       case BASIC:
         populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
 
         if (sub.getConsumers() != null) {
-          traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size());
+          traceDetails.put("numberOfConsumers", sub.getConsumers().size());
           traceDetails.put(
-              "Subscription.namesOfConsumers",
+              "namesOfConsumers",
               sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
         }
 
@@ -174,7 +211,7 @@ private static void populateSubscriptionDetails(
         populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
         populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
 
-        traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties());
+        traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -182,36 +219,45 @@ private static void populateSubscriptionDetails(
     }
   }
 
+  public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+    if (consumer == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateConsumerDetails(level, consumer, details);
+    return details;
+  }
+
   private static void populateConsumerDetails(
       TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
     if (consumer == null) {
-      traceDetails.put("Consumer", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("Consumer.name", consumer.consumerName());
-        traceDetails.put("Consumer.consumerId", consumer.consumerId());
+        traceDetails.put("name", consumer.consumerName());
+        traceDetails.put("consumerId", consumer.consumerId());
         if (consumer.getSubscription() != null) {
-          traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName());
+          traceDetails.put("subscriptionName", consumer.getSubscription().getName());
           traceDetails.put(
-              "Consumer.topicName",
+              "topicName",
               TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
         }
         break;
       case BASIC:
         populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
 
-        traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel());
-        traceDetails.put("Consumer.subType", consumer.subType().name());
-        traceDetails.put("Consumer.clientAddress", consumer.getClientAddress());
+        traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+        traceDetails.put("subType", consumer.subType().name());
+        traceDetails.put("clientAddress", consumer.getClientAddress());
         break;
       case FULL:
         populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
         populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
 
-        traceDetails.put("Consumer.metadata", consumer.getMetadata());
+        traceDetails.put("metadata", consumer.getMetadata());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -219,38 +265,46 @@ private static void populateConsumerDetails(
     }
   }
 
+  public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+    if (producer == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateProducerDetails(level, producer, details);
+    return details;
+  }
+
   private static void populateProducerDetails(
       TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
     if (producer == null) {
-      traceDetails.put("Producer", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("Producer.producerId", producer.getProducerId());
-        traceDetails.put("Producer.producerName", producer.getProducerName());
-        traceDetails.put("Producer.accessMode", producer.getAccessMode().name());
+        traceDetails.put("producerId", producer.getProducerId());
+        traceDetails.put("producerName", producer.getProducerName());
+        traceDetails.put("accessMode", producer.getAccessMode().name());
         if (producer.getTopic() != null) {
           traceDetails.put(
-              "Producer.topicName",
-              TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+              "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
         }
         break;
       case BASIC:
         populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
 
-        traceDetails.put("Producer.clientAddress", producer.getClientAddress());
+        traceDetails.put("clientAddress", producer.getClientAddress());
         break;
       case FULL:
         populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
         populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
 
-        traceDetails.put("Producer.metadata", producer.getMetadata());
+        traceDetails.put("metadata", producer.getMetadata());
         if (producer.getSchemaVersion() != null) {
-          traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString());
+          traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
         }
-        traceDetails.put("producer.remoteCluster", producer.getRemoteCluster());
+        traceDetails.put("remoteCluster", producer.getRemoteCluster());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -258,34 +312,44 @@ private static void populateProducerDetails(
     }
   }
 
+  public static Map<String, Object> getMessageMetadataDetails(
+      TraceLevel level, MessageMetadata msgMetadata) {
+    if (msgMetadata == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateMessageMetadataDetails(level, msgMetadata, details);
+    return details;
+  }
+
   private static void populateMessageMetadataDetails(
       TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
     if (msgMetadata == null) {
-      traceDetails.put("MessageMetadata", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId());
-        traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName());
-        traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey());
+        traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+        traceDetails.put("producerName", msgMetadata.getProducerName());
+        traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
         break;
       case BASIC:
         populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
 
-        traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize());
-        traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize());
-        traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+        traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+        traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+        traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
         break;
       case FULL:
         populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
         populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
 
-        traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime());
-        traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime());
-        traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom());
-        traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid());
+        traceDetails.put("publishTime", msgMetadata.getPublishTime());
+        traceDetails.put("eventTime", msgMetadata.getEventTime());
+        traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+        traceDetails.put("uuid", msgMetadata.getUuid());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -293,28 +357,37 @@ private static void populateMessageMetadataDetails(
     }
   }
 
+  public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+    if (entry == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateEntryDetails(level, entry, details);
+    return details;
+  }
+
   private static void populateEntryDetails(
       TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
     if (entry == null) {
-      traceDetails.put("Entry", "null");
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("Entry.ledgerId", entry.getLedgerId());
-        traceDetails.put("Entry.entryId", entry.getEntryId());
+        traceDetails.put("ledgerId", entry.getLedgerId());
+        traceDetails.put("entryId", entry.getEntryId());
         break;
       case BASIC:
         populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
 
-        traceDetails.put("Entry.length", entry.getLength());
+        traceDetails.put("length", entry.getLength());
         break;
       case FULL:
         populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
         populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
 
-        traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails);
+        traceByteBuf("data", entry.getDataBuffer(), traceDetails);
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -322,19 +395,28 @@ private static void populateEntryDetails(
     }
   }
 
+  public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+    if (publishContext == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populatePublishContext(publishContext, details);
+    return details;
+  }
+
   private static void populatePublishContext(
       Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
-    traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage());
-    traceDetails.put("publishContext.isChunked", publishContext.isChunked());
-    traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages());
+    traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+    traceDetails.put("isChunked", publishContext.isChunked());
+    traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
 
-    traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp());
-    traceDetails.put("publishContext.msgSize", publishContext.getMsgSize());
-    traceDetails.put("publishContext.producerName", publishContext.getProducerName());
-    traceDetails.put(
-        "publishContext.originalProducerName", publishContext.getOriginalProducerName());
-    traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId());
-    traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId());
+    traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
+    traceDetails.put("msgSize", publishContext.getMsgSize());
+    traceDetails.put("producerName", publishContext.getProducerName());
+    traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+    traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+    traceDetails.put("sequenceId", publishContext.getSequenceId());
   }
 
   private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
@@ -399,8 +481,8 @@ public void onConnectionCreated(ServerCnx cnx) {
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    tracer.info("Connection created: {}", traceDetails);
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    trace("Connection created", traceDetails);
   }
 
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -410,12 +492,11 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateProducerDetails(level, producer, traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("producer", getProducerDetails(level, producer));
     traceDetails.put("metadata", metadata);
 
-    tracer.info("Producer created: {}", traceDetails);
+    trace("Producer created", traceDetails);
   }
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
@@ -425,12 +506,11 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateProducerDetails(level, producer, traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("producer", getProducerDetails(level, producer));
     traceDetails.put("metadata", metadata);
 
-    tracer.info("Producer closed: {}", traceDetails);
+    trace("Producer closed", traceDetails);
   }
 
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -440,13 +520,12 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateConsumerDetails(level, consumer, traceDetails);
-    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("consumer", getConsumerDetails(level, consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
-    tracer.info("Consumer created: {}", traceDetails);
+    trace("Consumer created", traceDetails);
   }
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
@@ -456,13 +535,12 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateConsumerDetails(level, consumer, traceDetails);
-    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("consumer", getConsumerDetails(level, consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
-    tracer.info("Consumer closed: {}", traceDetails);
+    trace("Consumer closed", traceDetails);
   }
 
   public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
@@ -472,9 +550,10 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
     traceDetails.put("command", command.toString());
-    tracer.info("Pulsar command called: {}", traceDetails);
+
+    trace("Pulsar command called", traceDetails);
   }
 
   public void onConnectionClosed(ServerCnx cnx) {
@@ -484,8 +563,9 @@ public void onConnectionClosed(ServerCnx cnx) {
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    tracer.info("Connection closed: {}", traceDetails);
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+
+    trace("Connection closed", traceDetails);
   }
 
   /* ***************************
@@ -505,12 +585,12 @@ public void beforeSendMessage(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConsumerDetails(level, consumer, traceDetails);
-    populateSubscriptionDetails(level, subscription, traceDetails);
-    populateEntryDetails(level, entry, traceDetails);
-    populateMessageMetadataDetails(level, msgMetadata, traceDetails);
+    traceDetails.put("subscription", getSubscriptionDetails(level, subscription));
+    traceDetails.put("consumer", getConsumerDetails(level, consumer));
+    traceDetails.put("entry", getEntryDetails(level, entry));
+    traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
 
-    tracer.info("Before sending message: {}", traceDetails);
+    trace("Before sending message", traceDetails);
   }
 
   public void onMessagePublish(
@@ -523,13 +603,11 @@ public void onMessagePublish(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateProducerDetails(level, producer, traceDetails);
-
+    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
 
-    populatePublishContext(publishContext, traceDetails);
-
-    tracer.info("Message publish: {}", traceDetails);
+    trace("Message publish", traceDetails);
   }
 
   public void messageProduced(
@@ -545,16 +623,14 @@ public void messageProduced(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateProducerDetails(level, producer, traceDetails);
-    populatePublishContext(publishContext, traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("ledgerId", ledgerId);
     traceDetails.put("entryId", entryId);
     traceDetails.put("startTimeNs", startTimeNs);
 
-    tracer.info("Message produced: {}", traceDetails);
+    trace("Message produced", traceDetails);
   }
 
   public void messageDispatched(
@@ -565,16 +641,14 @@ public void messageDispatched(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateConsumerDetails(level, consumer, traceDetails);
-    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("consumer", getConsumerDetails(level, consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
     traceDetails.put("ledgerId", ledgerId);
     traceDetails.put("entryId", entryId);
-
     traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
 
-    tracer.info("After dispatching message: {}", traceDetails);
+    trace("After dispatching message", traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -584,21 +658,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    populateConnectionDetails(level, cnx, traceDetails);
-    populateConsumerDetails(level, consumer, traceDetails);
-    populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails);
-
-    traceDetails.put("ack.type", ackCmd.getAckType().name());
-    traceDetails.put("ack.consumerId", ackCmd.getConsumerId());
-    traceDetails.put(
-        "ack.messageIds",
+    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("consumer", getConsumerDetails(level, consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+
+    Map<String, Object> ackDetails = new TreeMap<>();
+    ackDetails.put("type", ackCmd.getAckType().name());
+    ackDetails.put("consumerId", ackCmd.getConsumerId());
+    ackDetails.put(
+        "messageIds",
         ackCmd
             .getMessageIdsList()
             .stream()
             .map(x -> x.getLedgerId() + ":" + x.getEntryId())
             .collect(Collectors.toList()));
 
-    tracer.info("Message acked: {}", traceDetails);
+    traceDetails.put("ack", ackDetails);
+
+    trace("Message acked", traceDetails);
   }
 
   /* ***************************
@@ -611,7 +688,7 @@ public void txnOpened(long tcId, String txnID) {
     traceDetails.put("tcId", tcId);
     traceDetails.put("txnID", txnID);
 
-    tracer.info("Transaction opened: {}", traceDetails);
+    trace("Transaction opened", traceDetails);
     //    }
   }
 
@@ -621,7 +698,7 @@ public void txnEnded(String txnID, long txnAction) {
     traceDetails.put("txnID", txnID);
     traceDetails.put("txnAction", txnAction);
 
-    tracer.info("Transaction closed: {}", traceDetails);
+    trace("Transaction closed", traceDetails);
     //    }
   }
 

From 40f47848b7de41987ffc56df93e14dd149c0a886 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 15:58:49 -0700
Subject: [PATCH 04/29] refactoring, cache some of configuration

---
 pom.xml                                       |   6 +
 pulsar-jms-tracing/pom.xml                    |   4 +
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 436 ++++--------------
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 361 +++++++++++++++
 4 files changed, 449 insertions(+), 358 deletions(-)
 create mode 100644 pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java

diff --git a/pom.xml b/pom.xml
index a24e298a..0b10c2b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,6 +65,7 @@
     <surefire.version>3.1.0</surefire.version>
     <jackson.version>2.14.2</jackson.version>
     <gson.version>2.8.9</gson.version>
+    <guava.version>32.1.2-jre</guava.version>
     <commons-compress.version>1.21</commons-compress.version>
     <awaitility.version>4.0.3</awaitility.version>
     <maven.antrun.plugin.version>3.1.0</maven.antrun.plugin.version>
@@ -107,6 +108,11 @@
         <artifactId>slf4j-simple</artifactId>
         <version>${slf4j.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>${guava.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.junit.jupiter</groupId>
         <artifactId>junit-jupiter-engine</artifactId>
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index c60c0b4a..da7d2dda 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -34,6 +34,10 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dd41a4c4..d3b7f248 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,22 +15,33 @@
  */
 package com.datastax.oss.pulsar.jms.tracing;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.Entry;
-import org.apache.commons.codec.binary.Hex;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.intercept.BrokerInterceptor;
 import org.apache.pulsar.broker.service.Consumer;
@@ -42,19 +53,60 @@
 import org.apache.pulsar.common.api.proto.CommandAck;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.intercept.InterceptException;
-import org.apache.pulsar.common.naming.TopicName;
 
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
 
-  private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
-
-  private static final ObjectMapper mapper =
-      new ObjectMapper()
-          .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
-          .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
-
-  static final int MAX_DATA_LENGTH = 1024;
+  private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
+      CacheBuilder.newBuilder()
+          .maximumSize(10)
+          .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+          .build(
+              new CacheLoader<PulsarService, Set<EventReasons>>() {
+                public Set<EventReasons> load(PulsarService pulsar) {
+                  // dynamic config can get reloaded without service restart
+                  String events =
+                      pulsar
+                          .getConfiguration()
+                          .getProperties()
+                          .getProperty("jmsTracingEventList", "");
+                  log.debug("read jmsTracingEventList: {}", events);
+
+                  Set<EventReasons> enabledEvents = new HashSet<>();
+
+                  for (String event : events.split(",")) {
+                    try {
+                      enabledEvents.add(EventReasons.valueOf(event.trim()));
+                    } catch (IllegalArgumentException e) {
+                      log.warn("Invalid event: {}. Skipping", event);
+                    }
+                  }
+
+                  return enabledEvents;
+                }
+              });
+
+  private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
+      CacheBuilder.newBuilder()
+          .maximumSize(10)
+          .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+          .build(
+              new CacheLoader<PulsarService, TraceLevel>() {
+                public TraceLevel load(PulsarService pulsar) {
+                  String level =
+                      pulsar
+                          .getConfiguration()
+                          .getProperties()
+                          .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+                  try {
+                    return TraceLevel.valueOf(level);
+                  } catch (IllegalArgumentException e) {
+                    log.warn(
+                        "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+                    return defaultTraceLevel;
+                  }
+                }
+              });
 
   public enum EventReasons {
     ADMINISTRATIVE,
@@ -63,372 +115,40 @@ public enum EventReasons {
     SERVLET,
   }
 
-  public enum TraceLevel {
-    NONE,
-    MINIMAL,
-    BASIC,
-    FULL
-  }
-
-  //  private final Set<EventReasons> defaultEnabledEvents = new HashSet<>();
   private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
 
-  public static void trace(String message, Map<String, Object> traceDetails) {
-    Map<String, Object> trace = new TreeMap<>();
-    trace.put("message", message);
-    trace.put("details", traceDetails);
-
-    try {
-      String loggableJsonString = mapper.writeValueAsString(trace);
-      tracer.info(loggableJsonString);
-    } catch (JsonProcessingException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   public void initialize(PulsarService pulsarService) {
-    //    defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE);
-    //    defaultEnabledEvents.add(EventReasons.MESSAGE);
-    //    defaultEnabledEvents.add(EventReasons.TRANSACTION);
+    log.info("Initializing BrokerTracing");
   }
 
   @Override
-  public void close() {}
+  public void close() {
+    log.info("Closing BrokerTracing");
+  }
 
   private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
     return getEnabledEvents(cnx.getBrokerService().getPulsar());
   }
 
   private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
-    // todo: do this less frequently
-
-    String events =
-        pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
-
-    Set<EventReasons> enabledEvents = new HashSet<>();
-
-    for (String event : events.split(",")) {
-      try {
-        enabledEvents.add(EventReasons.valueOf(event.trim()));
-      } catch (IllegalArgumentException e) {
-        log.warn("Invalid event: {}. Skipping", event);
-      }
+    try {
+      return jmsTracingEventList.get(pulsar);
+    } catch (ExecutionException e) {
+      log.error("Error getting enabled events", e);
+      return new HashSet<>();
     }
-
-    return enabledEvents;
   }
 
   private static TraceLevel getTracingLevel(ServerCnx cnx) {
-    // todo: do this less frequently
-    String level =
-        cnx.getBrokerService()
-            .getPulsar()
-            .getConfiguration()
-            .getProperties()
-            .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+
     try {
-      return TraceLevel.valueOf(level);
-    } catch (IllegalArgumentException e) {
-      log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+      return traceLevelForService.get(cnx.getBrokerService().getPulsar());
+    } catch (ExecutionException e) {
+      log.error("Error getting tracing level", e);
       return defaultTraceLevel;
     }
   }
 
-  public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
-    if (cnx == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateConnectionDetails(level, cnx, details);
-    return details;
-  }
-
-  private static void populateConnectionDetails(
-      TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
-    if (cnx == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("clientAddress", cnx.clientAddress().toString());
-        break;
-      case BASIC:
-        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
-
-        traceDetails.put("clientVersion", cnx.getClientVersion());
-        traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
-        break;
-      case FULL:
-        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
-        populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
-
-        traceDetails.put("authRole", cnx.getAuthRole());
-        traceDetails.put("authMethod", cnx.getAuthMethod());
-        traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
-    if (sub == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateSubscriptionDetails(level, sub, details);
-    return details;
-  }
-
-  private static void populateSubscriptionDetails(
-      TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
-    if (sub == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("name", sub.getName());
-        traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-        traceDetails.put("type", sub.getType().name());
-        break;
-      case BASIC:
-        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
-
-        if (sub.getConsumers() != null) {
-          traceDetails.put("numberOfConsumers", sub.getConsumers().size());
-          traceDetails.put(
-              "namesOfConsumers",
-              sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
-        }
-
-        break;
-      case FULL:
-        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
-        populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
-
-        traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
-    if (consumer == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateConsumerDetails(level, consumer, details);
-    return details;
-  }
-
-  private static void populateConsumerDetails(
-      TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
-    if (consumer == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("name", consumer.consumerName());
-        traceDetails.put("consumerId", consumer.consumerId());
-        if (consumer.getSubscription() != null) {
-          traceDetails.put("subscriptionName", consumer.getSubscription().getName());
-          traceDetails.put(
-              "topicName",
-              TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
-        }
-        break;
-      case BASIC:
-        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
-
-        traceDetails.put("priorityLevel", consumer.getPriorityLevel());
-        traceDetails.put("subType", consumer.subType().name());
-        traceDetails.put("clientAddress", consumer.getClientAddress());
-        break;
-      case FULL:
-        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
-        populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
-
-        traceDetails.put("metadata", consumer.getMetadata());
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
-    if (producer == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateProducerDetails(level, producer, details);
-    return details;
-  }
-
-  private static void populateProducerDetails(
-      TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
-    if (producer == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("producerId", producer.getProducerId());
-        traceDetails.put("producerName", producer.getProducerName());
-        traceDetails.put("accessMode", producer.getAccessMode().name());
-        if (producer.getTopic() != null) {
-          traceDetails.put(
-              "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
-        }
-        break;
-      case BASIC:
-        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
-
-        traceDetails.put("clientAddress", producer.getClientAddress());
-        break;
-      case FULL:
-        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
-        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
-
-        traceDetails.put("metadata", producer.getMetadata());
-        if (producer.getSchemaVersion() != null) {
-          traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
-        }
-        traceDetails.put("remoteCluster", producer.getRemoteCluster());
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getMessageMetadataDetails(
-      TraceLevel level, MessageMetadata msgMetadata) {
-    if (msgMetadata == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateMessageMetadataDetails(level, msgMetadata, details);
-    return details;
-  }
-
-  private static void populateMessageMetadataDetails(
-      TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
-    if (msgMetadata == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("sequenceId", msgMetadata.getSequenceId());
-        traceDetails.put("producerName", msgMetadata.getProducerName());
-        traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
-        break;
-      case BASIC:
-        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
-
-        traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
-        traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
-        traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
-        break;
-      case FULL:
-        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
-        populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
-
-        traceDetails.put("publishTime", msgMetadata.getPublishTime());
-        traceDetails.put("eventTime", msgMetadata.getEventTime());
-        traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
-        traceDetails.put("uuid", msgMetadata.getUuid());
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
-    if (entry == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateEntryDetails(level, entry, details);
-    return details;
-  }
-
-  private static void populateEntryDetails(
-      TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
-    if (entry == null) {
-      return;
-    }
-
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("ledgerId", entry.getLedgerId());
-        traceDetails.put("entryId", entry.getEntryId());
-        break;
-      case BASIC:
-        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
-
-        traceDetails.put("length", entry.getLength());
-        break;
-      case FULL:
-        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
-        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
-
-        traceByteBuf("data", entry.getDataBuffer(), traceDetails);
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
-  }
-
-  public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
-    if (publishContext == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populatePublishContext(publishContext, details);
-    return details;
-  }
-
-  private static void populatePublishContext(
-      Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
-    traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
-    traceDetails.put("isChunked", publishContext.isChunked());
-    traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
-
-    traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
-    traceDetails.put("msgSize", publishContext.getMsgSize());
-    traceDetails.put("producerName", publishContext.getProducerName());
-    traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
-    traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
-    traceDetails.put("sequenceId", publishContext.getSequenceId());
-  }
-
-  private static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
-    if (buf == null) return;
-
-    if (buf.readableBytes() < MAX_DATA_LENGTH) {
-      traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
-    } else {
-      traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
-    }
-  }
-
   private static TraceLevel getTracingLevel(Subscription sub) {
     if (sub == null
         || sub.getSubscriptionProperties() == null
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
new file mode 100644
index 00000000..c875e27e
--- /dev/null
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -0,0 +1,361 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datastax.oss.pulsar.jms.tracing;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import io.netty.buffer.ByteBuf;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.broker.service.Producer;
+import org.apache.pulsar.broker.service.ServerCnx;
+import org.apache.pulsar.broker.service.Subscription;
+import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.naming.TopicName;
+
+@Slf4j
+public class TracingUtils {
+  private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+  private static final ObjectMapper mapper =
+      new ObjectMapper()
+          .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
+          .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
+
+  public static final int MAX_DATA_LENGTH = 1024;
+
+  public enum TraceLevel {
+    NONE,
+    MINIMAL,
+    BASIC,
+    FULL
+  }
+
+  public static void trace(String message, Map<String, Object> traceDetails) {
+    Map<String, Object> trace = new TreeMap<>();
+    trace.put("message", message);
+    trace.put("details", traceDetails);
+
+    try {
+      String loggableJsonString = mapper.writeValueAsString(trace);
+      tracer.info(loggableJsonString);
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+    if (cnx == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateConnectionDetails(level, cnx, details);
+    return details;
+  }
+
+  private static void populateConnectionDetails(
+      TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+    if (cnx == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("clientAddress", cnx.clientAddress().toString());
+        break;
+      case BASIC:
+        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
+
+        traceDetails.put("clientVersion", cnx.getClientVersion());
+        traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+        break;
+      case FULL:
+        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
+        populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
+
+        traceDetails.put("authRole", cnx.getAuthRole());
+        traceDetails.put("authMethod", cnx.getAuthMethod());
+        traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+    if (sub == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateSubscriptionDetails(level, sub, details);
+    return details;
+  }
+
+  private static void populateSubscriptionDetails(
+      TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+    if (sub == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("name", sub.getName());
+        traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+        traceDetails.put("type", sub.getType().name());
+        break;
+      case BASIC:
+        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
+
+        if (sub.getConsumers() != null) {
+          traceDetails.put("numberOfConsumers", sub.getConsumers().size());
+          traceDetails.put(
+              "namesOfConsumers",
+              sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
+        }
+
+        break;
+      case FULL:
+        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
+        populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
+
+        traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+    if (consumer == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateConsumerDetails(level, consumer, details);
+    return details;
+  }
+
+  private static void populateConsumerDetails(
+      TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+    if (consumer == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("name", consumer.consumerName());
+        traceDetails.put("consumerId", consumer.consumerId());
+        if (consumer.getSubscription() != null) {
+          traceDetails.put("subscriptionName", consumer.getSubscription().getName());
+          traceDetails.put(
+              "topicName",
+              TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+        }
+        break;
+      case BASIC:
+        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+
+        traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+        traceDetails.put("subType", consumer.subType().name());
+        traceDetails.put("clientAddress", consumer.getClientAddress());
+        break;
+      case FULL:
+        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+        populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+
+        traceDetails.put("metadata", consumer.getMetadata());
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+    if (producer == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateProducerDetails(level, producer, details);
+    return details;
+  }
+
+  private static void populateProducerDetails(
+      TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+    if (producer == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("producerId", producer.getProducerId());
+        traceDetails.put("producerName", producer.getProducerName());
+        traceDetails.put("accessMode", producer.getAccessMode().name());
+        if (producer.getTopic() != null) {
+          traceDetails.put(
+              "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+        }
+        break;
+      case BASIC:
+        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+
+        traceDetails.put("clientAddress", producer.getClientAddress());
+        break;
+      case FULL:
+        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+
+        traceDetails.put("metadata", producer.getMetadata());
+        if (producer.getSchemaVersion() != null) {
+          traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
+        }
+        traceDetails.put("remoteCluster", producer.getRemoteCluster());
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getMessageMetadataDetails(
+      TraceLevel level, MessageMetadata msgMetadata) {
+    if (msgMetadata == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateMessageMetadataDetails(level, msgMetadata, details);
+    return details;
+  }
+
+  private static void populateMessageMetadataDetails(
+      TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+    if (msgMetadata == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+        traceDetails.put("producerName", msgMetadata.getProducerName());
+        traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+        break;
+      case BASIC:
+        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
+
+        traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+        traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+        traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+        break;
+      case FULL:
+        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
+        populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
+
+        traceDetails.put("publishTime", msgMetadata.getPublishTime());
+        traceDetails.put("eventTime", msgMetadata.getEventTime());
+        traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+        traceDetails.put("uuid", msgMetadata.getUuid());
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+    if (entry == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateEntryDetails(level, entry, details);
+    return details;
+  }
+
+  private static void populateEntryDetails(
+      TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+    if (entry == null) {
+      return;
+    }
+
+    switch (level) {
+      case MINIMAL:
+        traceDetails.put("ledgerId", entry.getLedgerId());
+        traceDetails.put("entryId", entry.getEntryId());
+        break;
+      case BASIC:
+        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+
+        traceDetails.put("length", entry.getLength());
+        break;
+      case FULL:
+        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+
+        traceByteBuf("data", entry.getDataBuffer(), traceDetails);
+        break;
+      default:
+        log.warn("Unknown tracing level: {}", level);
+        break;
+    }
+  }
+
+  public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+    if (publishContext == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populatePublishContext(publishContext, details);
+    return details;
+  }
+
+  private static void populatePublishContext(
+      Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+    traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+    traceDetails.put("isChunked", publishContext.isChunked());
+    traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+
+    traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
+    traceDetails.put("msgSize", publishContext.getMsgSize());
+    traceDetails.put("producerName", publishContext.getProducerName());
+    traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+    traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+    traceDetails.put("sequenceId", publishContext.getSequenceId());
+  }
+
+  public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
+    if (buf == null) return;
+
+    if (buf.readableBytes() < MAX_DATA_LENGTH) {
+      traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+    } else {
+      traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+    }
+  }
+}

From 2246bc44efc58596f8976f255f85e8cdf40cb8eb Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 16:42:36 -0700
Subject: [PATCH 05/29] more config caching

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 101 ++++++++++++------
 1 file changed, 70 insertions(+), 31 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index d3b7f248..3e0b91b8 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -36,6 +36,7 @@
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
@@ -57,10 +58,19 @@
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
 
+  public enum EventReasons {
+    ADMINISTRATIVE,
+    MESSAGE,
+    TRANSACTION,
+    SERVLET,
+  }
+
+  private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+
   private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
       CacheBuilder.newBuilder()
           .maximumSize(10)
-          .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+          .expireAfterWrite(1, TimeUnit.MINUTES)
           .build(
               new CacheLoader<PulsarService, Set<EventReasons>>() {
                 public Set<EventReasons> load(PulsarService pulsar) {
@@ -89,7 +99,7 @@ public Set<EventReasons> load(PulsarService pulsar) {
   private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
       CacheBuilder.newBuilder()
           .maximumSize(10)
-          .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES)
+          .expireAfterWrite(1, TimeUnit.MINUTES)
           .build(
               new CacheLoader<PulsarService, TraceLevel>() {
                 public TraceLevel load(PulsarService pulsar) {
@@ -108,14 +118,50 @@ public TraceLevel load(PulsarService pulsar) {
                 }
               });
 
-  public enum EventReasons {
-    ADMINISTRATIVE,
-    MESSAGE,
-    TRANSACTION,
-    SERVLET,
-  }
+  private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
+      CacheBuilder.newBuilder()
+          .expireAfterWrite(10, TimeUnit.SECONDS)
+          .build(
+              new CacheLoader<Subscription, TraceLevel>() {
+                public TraceLevel load(Subscription sub) {
+                  if (sub.getSubscriptionProperties() == null
+                      || !sub.getSubscriptionProperties().containsKey("trace")) {
+                    return TraceLevel.NONE;
+                  }
 
-  private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+                  try {
+                    return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+                  } catch (IllegalArgumentException e) {
+                    log.warn(
+                        "Invalid tracing level: {}. Setting to NONE for subscription {}",
+                        sub.getSubscriptionProperties().get("trace"),
+                        sub);
+                    return TraceLevel.NONE;
+                  }
+                }
+              });
+
+  private static final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
+      CacheBuilder.newBuilder()
+          .expireAfterWrite(10, TimeUnit.SECONDS)
+          .build(
+              new CacheLoader<Producer, TraceLevel>() {
+                public TraceLevel load(Producer producer) {
+                  if (producer.getMetadata() == null
+                      || !producer.getMetadata().containsKey("trace")) {
+                    return TraceLevel.NONE;
+                  }
+                  try {
+                    return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+                  } catch (IllegalArgumentException e) {
+                    log.warn(
+                        "Invalid tracing level: {}. Setting to NONE for producer {}",
+                        producer.getMetadata().get("trace"),
+                        producer);
+                    return TraceLevel.NONE;
+                  }
+                }
+              });
 
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
@@ -127,10 +173,14 @@ public void close() {
   }
 
   private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
+    if (cnx == null) return new HashSet<>();
+
     return getEnabledEvents(cnx.getBrokerService().getPulsar());
   }
 
   private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
+    if (pulsar == null) return new HashSet<>();
+
     try {
       return jmsTracingEventList.get(pulsar);
     } catch (ExecutionException e) {
@@ -140,6 +190,7 @@ private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
   }
 
   private static TraceLevel getTracingLevel(ServerCnx cnx) {
+    if (cnx == null) return defaultTraceLevel;
 
     try {
       return traceLevelForService.get(cnx.getBrokerService().getPulsar());
@@ -150,35 +201,23 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) {
   }
 
   private static TraceLevel getTracingLevel(Subscription sub) {
-    if (sub == null
-        || sub.getSubscriptionProperties() == null
-        || !sub.getSubscriptionProperties().containsKey("trace")) {
-      return TraceLevel.NONE;
-    }
+    if (sub == null) return TraceLevel.NONE;
+
     try {
-      return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
-    } catch (IllegalArgumentException e) {
-      log.warn(
-          "Invalid tracing level: {}. Setting to NONE for subscription {}",
-          sub.getSubscriptionProperties().get("trace"),
-          sub);
+      return traceLevelForSubscription.get(sub);
+    } catch (ExecutionException e) {
+      log.error("Error getting tracing level", e);
       return TraceLevel.NONE;
     }
   }
 
   private static TraceLevel getTracingLevel(Producer producer) {
-    if (producer == null
-        || producer.getMetadata() == null
-        || !producer.getMetadata().containsKey("trace")) {
-      return TraceLevel.NONE;
-    }
+    if (producer == null) return TraceLevel.NONE;
+
     try {
-      return TraceLevel.valueOf(producer.getMetadata().get("trace"));
-    } catch (IllegalArgumentException e) {
-      log.warn(
-          "Invalid tracing level: {}. Setting to NONE for producer {}",
-          producer.getMetadata().get("trace"),
-          producer);
+      return traceLevelForProducer.get(producer);
+    } catch (ExecutionException e) {
+      log.error("Error getting tracing level", e);
       return TraceLevel.NONE;
     }
   }

From 4f84dc7dc795e09bfc8690caa8412b328dd47009 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 12 Apr 2024 17:21:29 -0700
Subject: [PATCH 06/29] couple of tests

---
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  22 +++-
 .../pulsar/jms/tracing/TracingUtilsTest.java  | 103 ++++++++++++++++++
 2 files changed, 120 insertions(+), 5 deletions(-)
 create mode 100644 pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index c875e27e..fbe4d88b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -35,7 +35,14 @@
 
 @Slf4j
 public class TracingUtils {
-  private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+  private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+  @FunctionalInterface
+  public interface Tracer {
+    void trace(String message);
+  }
+
+  public static final Tracer SLF4J_TRACER = traceLogger::info;
 
   private static final ObjectMapper mapper =
       new ObjectMapper()
@@ -51,14 +58,19 @@ public enum TraceLevel {
     FULL
   }
 
+
   public static void trace(String message, Map<String, Object> traceDetails) {
+    trace(SLF4J_TRACER, message, traceDetails);
+  }
+
+  public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
     Map<String, Object> trace = new TreeMap<>();
     trace.put("message", message);
-    trace.put("details", traceDetails);
+    trace.put("traceDetails", traceDetails);
 
     try {
       String loggableJsonString = mapper.writeValueAsString(trace);
-      tracer.info(loggableJsonString);
+      tracer.trace(loggableJsonString);
     } catch (JsonProcessingException e) {
       throw new RuntimeException(e);
     }
@@ -353,9 +365,9 @@ public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> tra
     if (buf == null) return;
 
     if (buf.readableBytes() < MAX_DATA_LENGTH) {
-      traceDetails.put(key, Hex.encodeHex(buf.nioBuffer()));
+      traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
     } else {
-      traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+      traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
     }
   }
 }
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
new file mode 100644
index 00000000..956034cf
--- /dev/null
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -0,0 +1,103 @@
+package com.datastax.oss.pulsar.jms.tracing;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+
+class TracingUtilsTest {
+
+    List<String> traces = new ArrayList<>();
+    private Tracer mockTracer = new Tracer() {
+        @Override
+        public void trace(String msg) {
+            traces.add(msg);
+        }
+    };
+
+    @Test
+    void traceTest() {
+        traces.clear();
+        trace(mockTracer, "msg", null);
+        assertEquals(1, traces.size());
+        assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+
+        Map<String, Object> map = new TreeMap<>();
+
+        traces.clear();
+        trace(mockTracer, "msg", map);
+        assertEquals(1, traces.size());
+        assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+
+        map.put("key1", "value1");
+
+        traces.clear();
+        trace(mockTracer, "msg", map);
+        assertEquals(1, traces.size());
+        assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+    }
+
+// todo:
+//    @Test
+//    void getConnectionDetailsTest() {
+//    }
+//
+//    @Test
+//    void getSubscriptionDetailsTest() {
+//    }
+//
+//    @Test
+//    void getConsumerDetailsTest() {
+//    }
+//
+//    @Test
+//    void getProducerDetails() {
+//    }
+//
+//    @Test
+//    void getMessageMetadataDetailsTest() {
+//    }
+//
+//    @Test
+//    void getEntryDetailsTest() {
+//    }
+//
+//    @Test
+//    void getPublishContextDetailsTest() {
+//    }
+
+    @Test
+    void traceByteBufTest() {
+        Map<String, Object> traceDetails = new TreeMap<>();
+
+        traceByteBuf("key", null, traceDetails);
+        assertEquals(0, traceDetails.size());
+
+        ByteBuf small = Unpooled.buffer(20);
+        for (int i = 0; i < 20; i++) {
+            small.writeByte(i);
+        }
+        traceByteBuf("key", small, traceDetails);
+        assertEquals(1, traceDetails.size());
+        assertEquals(42, ((String) traceDetails.get("key")).length());
+        assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+
+        ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
+        for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+            big.writeByte(i);
+        }
+
+        traceDetails.clear();
+        traceByteBuf("key", big, traceDetails);
+        assertEquals(1, traceDetails.size());
+        assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+        assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+    }
+}
\ No newline at end of file

From ffc604eeab31d91ecbca6563e026df8714669247 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 10:24:24 -0700
Subject: [PATCH 07/29] feedback

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 184 +++++++----------
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  13 +-
 .../pulsar/jms/tracing/TracingUtilsTest.java  | 189 ++++++++++--------
 3 files changed, 184 insertions(+), 202 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 3e0b91b8..a1d7062b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.common.api.proto.CommandAck;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.intercept.InterceptException;
+import org.jetbrains.annotations.NotNull;
 
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
@@ -67,56 +68,40 @@ public enum EventReasons {
 
   private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
 
-  private static final LoadingCache<PulsarService, Set<EventReasons>> jmsTracingEventList =
-      CacheBuilder.newBuilder()
-          .maximumSize(10)
-          .expireAfterWrite(1, TimeUnit.MINUTES)
-          .build(
-              new CacheLoader<PulsarService, Set<EventReasons>>() {
-                public Set<EventReasons> load(PulsarService pulsar) {
-                  // dynamic config can get reloaded without service restart
-                  String events =
-                      pulsar
-                          .getConfiguration()
-                          .getProperties()
-                          .getProperty("jmsTracingEventList", "");
-                  log.debug("read jmsTracingEventList: {}", events);
-
-                  Set<EventReasons> enabledEvents = new HashSet<>();
-
-                  for (String event : events.split(",")) {
-                    try {
-                      enabledEvents.add(EventReasons.valueOf(event.trim()));
-                    } catch (IllegalArgumentException e) {
-                      log.warn("Invalid event: {}. Skipping", event);
-                    }
-                  }
+  private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
+  private TraceLevel traceLevel = defaultTraceLevel;
+
+  private static Set<EventReasons> loadEnabledEvents(
+      PulsarService pulsarService, Set<EventReasons> enabledEvents) {
+    String events =
+        pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
+    log.debug("read jmsTracingEventList: {}", events);
+
+    for (String event : events.split(",")) {
+      try {
+        enabledEvents.add(EventReasons.valueOf(event.trim()));
+      } catch (IllegalArgumentException e) {
+        log.error("Invalid event: {}. Skipping", event);
+      }
+    }
 
-                  return enabledEvents;
-                }
-              });
+    return enabledEvents;
+  }
 
-  private static final LoadingCache<PulsarService, TraceLevel> traceLevelForService =
-      CacheBuilder.newBuilder()
-          .maximumSize(10)
-          .expireAfterWrite(1, TimeUnit.MINUTES)
-          .build(
-              new CacheLoader<PulsarService, TraceLevel>() {
-                public TraceLevel load(PulsarService pulsar) {
-                  String level =
-                      pulsar
-                          .getConfiguration()
-                          .getProperties()
-                          .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
-                  try {
-                    return TraceLevel.valueOf(level);
-                  } catch (IllegalArgumentException e) {
-                    log.warn(
-                        "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
-                    return defaultTraceLevel;
-                  }
-                }
-              });
+  @NotNull
+  private static TraceLevel getTraceLevel(PulsarService pulsar) {
+    String level =
+        pulsar
+            .getConfiguration()
+            .getProperties()
+            .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
+    try {
+      return TraceLevel.valueOf(level);
+    } catch (IllegalArgumentException e) {
+      log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
+      return defaultTraceLevel;
+    }
+  }
 
   private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
       CacheBuilder.newBuilder()
@@ -124,17 +109,17 @@ public TraceLevel load(PulsarService pulsar) {
           .build(
               new CacheLoader<Subscription, TraceLevel>() {
                 public TraceLevel load(Subscription sub) {
-                  if (sub.getSubscriptionProperties() == null
-                      || !sub.getSubscriptionProperties().containsKey("trace")) {
+                  Map<String, String> subProps = sub.getSubscriptionProperties();
+                  if (subProps == null || !subProps.containsKey("trace")) {
                     return TraceLevel.NONE;
                   }
 
                   try {
-                    return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace"));
+                    return TraceLevel.valueOf(subProps.get("trace"));
                   } catch (IllegalArgumentException e) {
                     log.warn(
                         "Invalid tracing level: {}. Setting to NONE for subscription {}",
-                        sub.getSubscriptionProperties().get("trace"),
+                        subProps.get("trace"),
                         sub);
                     return TraceLevel.NONE;
                   }
@@ -165,6 +150,9 @@ public TraceLevel load(Producer producer) {
 
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
+
+    loadEnabledEvents(pulsarService, jmsTracingEventList);
+    traceLevel = getTraceLevel(pulsarService);
   }
 
   @Override
@@ -172,32 +160,9 @@ public void close() {
     log.info("Closing BrokerTracing");
   }
 
-  private Set<EventReasons> getEnabledEvents(ServerCnx cnx) {
-    if (cnx == null) return new HashSet<>();
-
-    return getEnabledEvents(cnx.getBrokerService().getPulsar());
-  }
-
-  private Set<EventReasons> getEnabledEvents(PulsarService pulsar) {
-    if (pulsar == null) return new HashSet<>();
-
-    try {
-      return jmsTracingEventList.get(pulsar);
-    } catch (ExecutionException e) {
-      log.error("Error getting enabled events", e);
-      return new HashSet<>();
-    }
-  }
-
-  private static TraceLevel getTracingLevel(ServerCnx cnx) {
-    if (cnx == null) return defaultTraceLevel;
-
-    try {
-      return traceLevelForService.get(cnx.getBrokerService().getPulsar());
-    } catch (ExecutionException e) {
-      log.error("Error getting tracing level", e);
-      return defaultTraceLevel;
-    }
+  private static TraceLevel getTracingLevel(Consumer consumer) {
+    if (consumer == null) return TraceLevel.NONE;
+    return getTracingLevel(consumer.getSubscription());
   }
 
   private static TraceLevel getTracingLevel(Subscription sub) {
@@ -234,20 +199,19 @@ private static TraceLevel getTracingLevel(Producer producer) {
    ******************************/
 
   public void onConnectionCreated(ServerCnx cnx) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
     trace("Connection created", traceDetails);
   }
 
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
+    TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -259,9 +223,9 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
   }
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
+    TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -273,9 +237,9 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
   }
 
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
+    TraceLevel level = getTracingLevel(consumer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -288,9 +252,9 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
   }
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
+    TraceLevel level = getTracingLevel(consumer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -303,26 +267,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
   }
 
   public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
     traceDetails.put("command", command.toString());
 
     trace("Pulsar command called", traceDetails);
   }
 
   public void onConnectionClosed(ServerCnx cnx) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    TraceLevel level = getTracingLevel(cnx);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
 
     trace("Connection closed", traceDetails);
   }
@@ -337,8 +299,7 @@ public void beforeSendMessage(
       long[] ackSet,
       MessageMetadata msgMetadata,
       Consumer consumer) {
-    if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar())
-        .contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(subscription);
     if (level == TraceLevel.NONE) return;
@@ -355,8 +316,7 @@ public void beforeSendMessage(
   public void onMessagePublish(
       Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
 
-    if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar())
-        .contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.NONE) return;
@@ -376,7 +336,7 @@ public void messageProduced(
       long ledgerId,
       long entryId,
       Topic.PublishContext publishContext) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.NONE) return;
@@ -394,9 +354,9 @@ public void messageProduced(
 
   public void messageDispatched(
       ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
-    TraceLevel level = getTracingLevel(consumer.getSubscription());
+    TraceLevel level = getTracingLevel(consumer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -411,9 +371,9 @@ public void messageDispatched(
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
-    if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
-    TraceLevel level = getTracingLevel(consumer.getSubscription());
+    TraceLevel level = getTracingLevel(consumer);
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
@@ -442,23 +402,25 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
    ******************************/
 
   public void txnOpened(long tcId, String txnID) {
-    //    if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+    if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
+    if (traceLevel == TraceLevel.NONE) return;
+
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("tcId", tcId);
     traceDetails.put("txnID", txnID);
 
     trace("Transaction opened", traceDetails);
-    //    }
   }
 
   public void txnEnded(String txnID, long txnAction) {
-    //    if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) {
+    if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
+    if (traceLevel == TraceLevel.NONE) return;
+
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("txnID", txnID);
     traceDetails.put("txnAction", txnAction);
 
     trace("Transaction closed", traceDetails);
-    //    }
   }
 
   /* ***************************
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index fbe4d88b..bed9bf17 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -35,7 +35,8 @@
 
 @Slf4j
 public class TracingUtils {
-  private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing");
+  private static final org.slf4j.Logger traceLogger =
+      org.slf4j.LoggerFactory.getLogger("jms-tracing");
 
   @FunctionalInterface
   public interface Tracer {
@@ -58,7 +59,6 @@ public enum TraceLevel {
     FULL
   }
 
-
   public static void trace(String message, Map<String, Object> traceDetails) {
     trace(SLF4J_TRACER, message, traceDetails);
   }
@@ -72,7 +72,11 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
       String loggableJsonString = mapper.writeValueAsString(trace);
       tracer.trace(loggableJsonString);
     } catch (JsonProcessingException e) {
-      throw new RuntimeException(e);
+      log.error(
+          "Failed to serialize trace message '{}' as json, traceDetails: {}",
+          message,
+          traceDetails,
+          e);
     }
   }
 
@@ -367,7 +371,8 @@ public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> tra
     if (buf.readableBytes() < MAX_DATA_LENGTH) {
       traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
     } else {
-      traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+      traceDetails.put(
+          key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
     }
   }
 }
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 956034cf..4675de18 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -1,103 +1,118 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.datastax.oss.pulsar.jms.tracing;
 
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+import static org.junit.jupiter.api.Assertions.*;
+
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
-import org.junit.jupiter.api.Test;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*;
+import org.junit.jupiter.api.Test;
 
 class TracingUtilsTest {
 
-    List<String> traces = new ArrayList<>();
-    private Tracer mockTracer = new Tracer() {
+  List<String> traces = new ArrayList<>();
+  private Tracer mockTracer =
+      new Tracer() {
         @Override
         public void trace(String msg) {
-            traces.add(msg);
+          traces.add(msg);
         }
-    };
-
-    @Test
-    void traceTest() {
-        traces.clear();
-        trace(mockTracer, "msg", null);
-        assertEquals(1, traces.size());
-        assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
-
-        Map<String, Object> map = new TreeMap<>();
-
-        traces.clear();
-        trace(mockTracer, "msg", map);
-        assertEquals(1, traces.size());
-        assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
-
-        map.put("key1", "value1");
-
-        traces.clear();
-        trace(mockTracer, "msg", map);
-        assertEquals(1, traces.size());
-        assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+      };
+
+  @Test
+  void traceTest() {
+    traces.clear();
+    trace(mockTracer, "msg", null);
+    assertEquals(1, traces.size());
+    assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+
+    Map<String, Object> map = new TreeMap<>();
+
+    traces.clear();
+    trace(mockTracer, "msg", map);
+    assertEquals(1, traces.size());
+    assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+
+    map.put("key1", "value1");
+
+    traces.clear();
+    trace(mockTracer, "msg", map);
+    assertEquals(1, traces.size());
+    assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+  }
+
+  // todo:
+  //    @Test
+  //    void getConnectionDetailsTest() {
+  //    }
+  //
+  //    @Test
+  //    void getSubscriptionDetailsTest() {
+  //    }
+  //
+  //    @Test
+  //    void getConsumerDetailsTest() {
+  //    }
+  //
+  //    @Test
+  //    void getProducerDetails() {
+  //    }
+  //
+  //    @Test
+  //    void getMessageMetadataDetailsTest() {
+  //    }
+  //
+  //    @Test
+  //    void getEntryDetailsTest() {
+  //    }
+  //
+  //    @Test
+  //    void getPublishContextDetailsTest() {
+  //    }
+
+  @Test
+  void traceByteBufTest() {
+    Map<String, Object> traceDetails = new TreeMap<>();
+
+    traceByteBuf("key", null, traceDetails);
+    assertEquals(0, traceDetails.size());
+
+    ByteBuf small = Unpooled.buffer(20);
+    for (int i = 0; i < 20; i++) {
+      small.writeByte(i);
     }
-
-// todo:
-//    @Test
-//    void getConnectionDetailsTest() {
-//    }
-//
-//    @Test
-//    void getSubscriptionDetailsTest() {
-//    }
-//
-//    @Test
-//    void getConsumerDetailsTest() {
-//    }
-//
-//    @Test
-//    void getProducerDetails() {
-//    }
-//
-//    @Test
-//    void getMessageMetadataDetailsTest() {
-//    }
-//
-//    @Test
-//    void getEntryDetailsTest() {
-//    }
-//
-//    @Test
-//    void getPublishContextDetailsTest() {
-//    }
-
-    @Test
-    void traceByteBufTest() {
-        Map<String, Object> traceDetails = new TreeMap<>();
-
-        traceByteBuf("key", null, traceDetails);
-        assertEquals(0, traceDetails.size());
-
-        ByteBuf small = Unpooled.buffer(20);
-        for (int i = 0; i < 20; i++) {
-            small.writeByte(i);
-        }
-        traceByteBuf("key", small, traceDetails);
-        assertEquals(1, traceDetails.size());
-        assertEquals(42, ((String) traceDetails.get("key")).length());
-        assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
-
-        ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
-        for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
-            big.writeByte(i);
-        }
-
-        traceDetails.clear();
-        traceByteBuf("key", big, traceDetails);
-        assertEquals(1, traceDetails.size());
-        assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
-        assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+    traceByteBuf("key", small, traceDetails);
+    assertEquals(1, traceDetails.size());
+    assertEquals(42, ((String) traceDetails.get("key")).length());
+    assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+
+    ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
+    for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+      big.writeByte(i);
     }
-}
\ No newline at end of file
+
+    traceDetails.clear();
+    traceByteBuf("key", big, traceDetails);
+    assertEquals(1, traceDetails.size());
+    assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+    assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+  }
+}

From 4585bcab8266c7f3b337249dbc3e600bbaed6468 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 14:19:26 -0700
Subject: [PATCH 08/29] fixes, tweaks

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 12 ++--
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 62 ++++++++++++-------
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index a1d7062b..2b94a263 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -79,7 +79,7 @@ private static Set<EventReasons> loadEnabledEvents(
 
     for (String event : events.split(",")) {
       try {
-        enabledEvents.add(EventReasons.valueOf(event.trim()));
+        enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
       } catch (IllegalArgumentException e) {
         log.error("Invalid event: {}. Skipping", event);
       }
@@ -96,7 +96,7 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
             .getProperties()
             .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
     try {
-      return TraceLevel.valueOf(level);
+      return TraceLevel.valueOf(level.trim().toUpperCase());
     } catch (IllegalArgumentException e) {
       log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
       return defaultTraceLevel;
@@ -115,7 +115,7 @@ public TraceLevel load(Subscription sub) {
                   }
 
                   try {
-                    return TraceLevel.valueOf(subProps.get("trace"));
+                    return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
                   } catch (IllegalArgumentException e) {
                     log.warn(
                         "Invalid tracing level: {}. Setting to NONE for subscription {}",
@@ -137,7 +137,8 @@ public TraceLevel load(Producer producer) {
                     return TraceLevel.NONE;
                   }
                   try {
-                    return TraceLevel.valueOf(producer.getMetadata().get("trace"));
+                    return TraceLevel.valueOf(
+                        producer.getMetadata().get("trace").trim().toUpperCase());
                   } catch (IllegalArgumentException e) {
                     log.warn(
                         "Invalid tracing level: {}. Setting to NONE for producer {}",
@@ -273,6 +274,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+    // todo: .toString() is not good enough
+    // {"message":"Pulsar command
+    // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}}
     traceDetails.put("command", command.toString());
 
     trace("Pulsar command called", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index bed9bf17..372e6ca9 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -107,12 +107,15 @@ private static void populateConnectionDetails(
         traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
         break;
       case FULL:
-        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
         populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
 
         traceDetails.put("authRole", cnx.getAuthRole());
         traceDetails.put("authMethod", cnx.getAuthMethod());
-        traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+        traceDetails.put(
+            "authMethodName",
+            cnx.getAuthenticationProvider() == null
+                ? "no provider"
+                : cnx.getAuthenticationProvider().getAuthMethodName());
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -154,7 +157,6 @@ private static void populateSubscriptionDetails(
 
         break;
       case FULL:
-        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
         populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
 
         traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
@@ -185,22 +187,21 @@ private static void populateConsumerDetails(
       case MINIMAL:
         traceDetails.put("name", consumer.consumerName());
         traceDetails.put("consumerId", consumer.consumerId());
-        if (consumer.getSubscription() != null) {
-          traceDetails.put("subscriptionName", consumer.getSubscription().getName());
+        Subscription sub = consumer.getSubscription();
+        if (sub != null) {
+          traceDetails.put("subscriptionName", sub.getName());
           traceDetails.put(
-              "topicName",
-              TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName());
+              "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
         }
         break;
       case BASIC:
         populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
 
         traceDetails.put("priorityLevel", consumer.getPriorityLevel());
-        traceDetails.put("subType", consumer.subType().name());
+        traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
         traceDetails.put("clientAddress", consumer.getClientAddress());
         break;
       case FULL:
-        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
         populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
 
         traceDetails.put("metadata", consumer.getMetadata());
@@ -231,7 +232,9 @@ private static void populateProducerDetails(
       case MINIMAL:
         traceDetails.put("producerId", producer.getProducerId());
         traceDetails.put("producerName", producer.getProducerName());
-        traceDetails.put("accessMode", producer.getAccessMode().name());
+        traceDetails.put(
+            "accessMode",
+            producer.getAccessMode() == null ? null : producer.getAccessMode().name());
         if (producer.getTopic() != null) {
           traceDetails.put(
               "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
@@ -243,7 +246,6 @@ private static void populateProducerDetails(
         traceDetails.put("clientAddress", producer.getClientAddress());
         break;
       case FULL:
-        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
         populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
 
         traceDetails.put("metadata", producer.getMetadata());
@@ -277,25 +279,42 @@ private static void populateMessageMetadataDetails(
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("sequenceId", msgMetadata.getSequenceId());
-        traceDetails.put("producerName", msgMetadata.getProducerName());
-        traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+        if (msgMetadata.hasPartitionKey()) {
+          traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+        }
+        if (msgMetadata.hasSequenceId()) {
+          traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+        }
+        if (msgMetadata.hasProducerName()) {
+          traceDetails.put("producerName", msgMetadata.getProducerName());
+        }
         break;
       case BASIC:
         populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
 
-        traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+        if (msgMetadata.hasUncompressedSize()) {
+          traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+        }
+        if (msgMetadata.hasNumMessagesInBatch()) {
+          traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+        }
         traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
-        traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
         break;
       case FULL:
-        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
         populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
 
-        traceDetails.put("publishTime", msgMetadata.getPublishTime());
-        traceDetails.put("eventTime", msgMetadata.getEventTime());
-        traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
-        traceDetails.put("uuid", msgMetadata.getUuid());
+        if (msgMetadata.hasPublishTime()) {
+          traceDetails.put("publishTime", msgMetadata.getPublishTime());
+        }
+        if (msgMetadata.hasEventTime()) {
+          traceDetails.put("eventTime", msgMetadata.getEventTime());
+        }
+        if (msgMetadata.hasReplicatedFrom()) {
+          traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+        }
+        if (msgMetadata.hasUuid()) {
+          traceDetails.put("uuid", msgMetadata.getUuid());
+        }
         break;
       default:
         log.warn("Unknown tracing level: {}", level);
@@ -330,7 +349,6 @@ private static void populateEntryDetails(
         traceDetails.put("length", entry.getLength());
         break;
       case FULL:
-        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
         populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
 
         traceByteBuf("data", entry.getDataBuffer(), traceDetails);

From 5e9094a10a027e2d25711acb8c958cc2c2d0b169 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 15 Apr 2024 16:20:37 -0700
Subject: [PATCH 09/29] actually trace commands; 'message' changed to
 'eventType'

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java |  12 +-
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 301 +++++++++++++++++-
 2 files changed, 307 insertions(+), 6 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 2b94a263..7606563f 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -16,6 +16,7 @@
 package com.datastax.oss.pulsar.jms.tracing;
 
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
@@ -274,10 +275,13 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
-    // todo: .toString() is not good enough
-    // {"message":"Pulsar command
-    // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}}
-    traceDetails.put("command", command.toString());
+
+    if (command.hasType()) {
+      traceDetails.put("type", command.getType().name());
+      traceDetails.put("command", getCommandDetails(traceLevel, command));
+    } else {
+      traceDetails.put("type", "unknown/null");
+    }
 
     trace("Pulsar command called", traceDetails);
   }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 372e6ca9..cc6ba6de 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -19,7 +19,11 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import io.netty.buffer.ByteBuf;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.Map;
+import java.util.Optional;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -30,6 +34,7 @@
 import org.apache.pulsar.broker.service.ServerCnx;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.naming.TopicName;
 
@@ -65,7 +70,7 @@ public static void trace(String message, Map<String, Object> traceDetails) {
 
   public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
     Map<String, Object> trace = new TreeMap<>();
-    trace.put("message", message);
+    trace.put("eventType", message);
     trace.put("traceDetails", traceDetails);
 
     try {
@@ -73,13 +78,293 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
       tracer.trace(loggableJsonString);
     } catch (JsonProcessingException e) {
       log.error(
-          "Failed to serialize trace message '{}' as json, traceDetails: {}",
+          "Failed to serialize trace event type '{}' as json, traceDetails: {}",
           message,
           traceDetails,
           e);
     }
   }
 
+  public static Map<String, Object> getCommandDetails(TraceLevel level, BaseCommand command) {
+    if (command == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateCommandDetails(level, command, details);
+    return details;
+  }
+
+  private static void populateCommandDetails(
+      TraceLevel level, BaseCommand command, Map<String, Object> traceDetails) {
+    if (command == null) {
+      return;
+    }
+    if (level == TraceLevel.NONE) {
+      return;
+    }
+
+    if (!command.hasType()) {
+      return;
+    }
+
+    // trace all params otherwise
+
+    switch (command.getType()) {
+      case CONNECT:
+        populateByReflection(command.getConnect(), traceDetails);
+        break;
+      case CONNECTED:
+        populateByReflection(command.getConnected(), traceDetails);
+        break;
+      case SUBSCRIBE:
+        populateByReflection(command.getSubscribe(), traceDetails);
+        break;
+      case PRODUCER:
+        populateByReflection(command.getProducer(), traceDetails);
+        break;
+      case SEND:
+        populateByReflection(command.getSend(), traceDetails);
+        break;
+      case SEND_RECEIPT:
+        populateByReflection(command.getSendReceipt(), traceDetails);
+        break;
+      case SEND_ERROR:
+        populateByReflection(command.getSendError(), traceDetails);
+        break;
+      case MESSAGE:
+        populateByReflection(command.getMessage(), traceDetails);
+        break;
+      case ACK:
+        populateByReflection(command.getAck(), traceDetails);
+        break;
+      case FLOW:
+        populateByReflection(command.getFlow(), traceDetails);
+        break;
+      case UNSUBSCRIBE:
+        populateByReflection(command.getUnsubscribe(), traceDetails);
+        break;
+      case SUCCESS:
+        populateByReflection(command.getSuccess(), traceDetails);
+        break;
+      case ERROR:
+        populateByReflection(command.getError(), traceDetails);
+        break;
+      case CLOSE_PRODUCER:
+        populateByReflection(command.getCloseProducer(), traceDetails);
+        break;
+      case CLOSE_CONSUMER:
+        populateByReflection(command.getCloseConsumer(), traceDetails);
+        break;
+      case PRODUCER_SUCCESS:
+        populateByReflection(command.getProducerSuccess(), traceDetails);
+        break;
+      case PING:
+        populateByReflection(command.getPing(), traceDetails);
+        break;
+      case PONG:
+        populateByReflection(command.getPong(), traceDetails);
+        break;
+      case REDELIVER_UNACKNOWLEDGED_MESSAGES:
+        populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
+        break;
+      case PARTITIONED_METADATA:
+        populateByReflection(command.getPartitionMetadata(), traceDetails);
+        break;
+      case PARTITIONED_METADATA_RESPONSE:
+        populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
+        break;
+      case LOOKUP:
+        populateByReflection(command.getLookupTopic(), traceDetails);
+        break;
+      case LOOKUP_RESPONSE:
+        populateByReflection(command.getLookupTopicResponse(), traceDetails);
+        break;
+      case CONSUMER_STATS:
+        populateByReflection(command.getConsumerStats(), traceDetails);
+        break;
+      case CONSUMER_STATS_RESPONSE:
+        populateByReflection(command.getConsumerStatsResponse(), traceDetails);
+        break;
+      case REACHED_END_OF_TOPIC:
+        populateByReflection(command.getReachedEndOfTopic(), traceDetails);
+        break;
+      case SEEK:
+        populateByReflection(command.getSeek(), traceDetails);
+        break;
+      case GET_LAST_MESSAGE_ID:
+        populateByReflection(command.getGetLastMessageId(), traceDetails);
+        break;
+      case GET_LAST_MESSAGE_ID_RESPONSE:
+        populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
+        break;
+      case ACTIVE_CONSUMER_CHANGE:
+        populateByReflection(command.getActiveConsumerChange(), traceDetails);
+        break;
+      case GET_TOPICS_OF_NAMESPACE:
+        populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
+        break;
+      case GET_TOPICS_OF_NAMESPACE_RESPONSE:
+        populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
+        break;
+      case GET_SCHEMA:
+        populateByReflection(command.getGetSchema(), traceDetails);
+        break;
+      case GET_SCHEMA_RESPONSE:
+        populateByReflection(command.getGetSchemaResponse(), traceDetails);
+        break;
+      case AUTH_CHALLENGE:
+        populateByReflection(command.getAuthChallenge(), traceDetails);
+        break;
+      case AUTH_RESPONSE:
+        populateByReflection(command.getAuthResponse(), traceDetails);
+        break;
+      case ACK_RESPONSE:
+        populateByReflection(command.getAckResponse(), traceDetails);
+        break;
+      case GET_OR_CREATE_SCHEMA:
+        populateByReflection(command.getGetOrCreateSchema(), traceDetails);
+        break;
+      case GET_OR_CREATE_SCHEMA_RESPONSE:
+        populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
+        break;
+      case NEW_TXN:
+        populateByReflection(command.getNewTxn(), traceDetails);
+        break;
+      case NEW_TXN_RESPONSE:
+        populateByReflection(command.getNewTxnResponse(), traceDetails);
+        break;
+      case ADD_PARTITION_TO_TXN:
+        populateByReflection(command.getAddPartitionToTxn(), traceDetails);
+        break;
+      case ADD_PARTITION_TO_TXN_RESPONSE:
+        populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
+        break;
+      case ADD_SUBSCRIPTION_TO_TXN:
+        populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
+        break;
+      case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
+        populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
+        break;
+      case END_TXN:
+        populateByReflection(command.getEndTxn(), traceDetails);
+        break;
+      case END_TXN_RESPONSE:
+        populateByReflection(command.getEndTxnResponse(), traceDetails);
+        break;
+      case END_TXN_ON_PARTITION:
+        populateByReflection(command.getEndTxnOnPartition(), traceDetails);
+        break;
+      case END_TXN_ON_PARTITION_RESPONSE:
+        populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
+        break;
+      case END_TXN_ON_SUBSCRIPTION:
+        populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
+        break;
+      case END_TXN_ON_SUBSCRIPTION_RESPONSE:
+        populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
+        break;
+      case TC_CLIENT_CONNECT_REQUEST:
+        populateByReflection(command.getTcClientConnectRequest(), traceDetails);
+        break;
+      case TC_CLIENT_CONNECT_RESPONSE:
+        populateByReflection(command.getTcClientConnectResponse(), traceDetails);
+        break;
+      case WATCH_TOPIC_LIST:
+        populateByReflection(command.getWatchTopicList(), traceDetails);
+        break;
+      case WATCH_TOPIC_LIST_SUCCESS:
+        populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
+        break;
+      case WATCH_TOPIC_UPDATE:
+        populateByReflection(command.getWatchTopicUpdate(), traceDetails);
+        break;
+      case WATCH_TOPIC_LIST_CLOSE:
+        populateByReflection(command.getWatchTopicListClose(), traceDetails);
+        break;
+      case TOPIC_MIGRATED:
+        populateByReflection(command.getTopicMigrated(), traceDetails);
+        break;
+      default:
+        log.error("Unknown command type: {}", command.getType());
+        traceDetails.put("error", "unknownCommandType " + command.getType());
+    }
+  }
+
+  private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
+    if (command == null) {
+      return;
+    }
+    if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) {
+      return;
+    }
+
+    Method[] allMethods = command.getClass().getMethods();
+
+    Arrays.stream(allMethods)
+        .filter(method -> method.getName().startsWith("has"))
+        .filter(
+            method -> {
+              try {
+                return (boolean) method.invoke(command);
+              } catch (Exception e) {
+                return false;
+              }
+            })
+        .forEach(
+            method -> {
+              String fieldName = method.getName().substring(3);
+              try {
+                Optional<Method> accessor =
+                    Arrays.stream(allMethods)
+                        .filter(
+                            m ->
+                                m.getName().equals("get" + fieldName)
+                                    || m.getName().equals("is" + fieldName))
+                        .findFirst();
+                if (!accessor.isPresent()) {
+                  log.warn(
+                      "No accessor found for field (but has.. counterpart was found): {} of {}",
+                      fieldName,
+                      command.getClass().getCanonicalName());
+                  return;
+                }
+                Object value = accessor.get().invoke(command);
+
+                if (value == null) return;
+
+                // skip logging of binary data
+                if (value instanceof byte[]
+                    || value instanceof ByteBuf
+                    || value instanceof ByteBuffer) {
+                  int size = 0;
+                  if (value instanceof byte[]) {
+                    size = ((byte[]) value).length;
+                  } else if (value instanceof ByteBuf) {
+                    size = ((ByteBuf) value).readableBytes();
+                  } else if (value instanceof ByteBuffer) {
+                    size = ((ByteBuffer) value).remaining();
+                  }
+                  traceDetails.put(fieldName + "_size", size);
+                  return;
+                }
+
+                if (value
+                    .getClass()
+                    .getCanonicalName()
+                    .contains("org.apache.pulsar.common.api.proto")) {
+                  Map<String, Object> details = new TreeMap<>();
+                  populateByReflection(value, details);
+                  traceDetails.put(fieldName, details);
+                } else {
+                  traceDetails.put(fieldName, value);
+                }
+              } catch (Exception e) {
+                log.error("Failed to access field: {}", fieldName, e);
+              }
+            });
+  }
+
   public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
     if (cnx == null) {
       return null;
@@ -117,6 +402,8 @@ private static void populateConnectionDetails(
                 ? "no provider"
                 : cnx.getAuthenticationProvider().getAuthMethodName());
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;
@@ -161,6 +448,8 @@ private static void populateSubscriptionDetails(
 
         traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;
@@ -206,6 +495,8 @@ private static void populateConsumerDetails(
 
         traceDetails.put("metadata", consumer.getMetadata());
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;
@@ -254,6 +545,8 @@ private static void populateProducerDetails(
         }
         traceDetails.put("remoteCluster", producer.getRemoteCluster());
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;
@@ -316,6 +609,8 @@ private static void populateMessageMetadataDetails(
           traceDetails.put("uuid", msgMetadata.getUuid());
         }
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;
@@ -353,6 +648,8 @@ private static void populateEntryDetails(
 
         traceByteBuf("data", entry.getDataBuffer(), traceDetails);
         break;
+      case NONE:
+        break;
       default:
         log.warn("Unknown tracing level: {}", level);
         break;

From f43457ffad0deb17a87fbd190a5afd48888b9d28 Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 11:40:26 +0200
Subject: [PATCH 10/29] Fix test

---
 .../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 4675de18..fa148cc8 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -42,21 +42,21 @@ void traceTest() {
     traces.clear();
     trace(mockTracer, "msg", null);
     assertEquals(1, traces.size());
-    assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0));
+    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
 
     Map<String, Object> map = new TreeMap<>();
 
     traces.clear();
     trace(mockTracer, "msg", map);
     assertEquals(1, traces.size());
-    assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
 
     map.put("key1", "value1");
 
     traces.clear();
     trace(mockTracer, "msg", map);
     assertEquals(1, traces.size());
-    assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
   }
 
   // todo:

From 1d5e7b87a23943ff8d360b60a23980c851073edc Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 11:45:31 +0200
Subject: [PATCH 11/29] add pulsar-jms-tracing to the release and upgrade the
 Nifi-plugin

---
 .github/workflows/release.yml        | 2 +-
 pulsar-jms-admin-ext/pom.xml         | 2 +-
 pulsar-jms-filters/pom.xml           | 2 +-
 pulsar-jms-integration-tests/pom.xml | 4 ++--
 pulsar-jms-tracing/pom.xml           | 2 +-
 pulsar-jms/pom.xml                   | 8 ++++----
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fc9e9cb1..2095b43c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -20,6 +20,6 @@ jobs:
         run: mvn -B package -DskipTests
       - uses: ncipollo/release-action@v1
         with:
-          artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar"
+          artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar,pulsar-jms-tracing/target/pulsar-jms*.nar"
           token: ${{ secrets.GITHUB_TOKEN }}
           generateReleaseNotes: true
diff --git a/pulsar-jms-admin-ext/pom.xml b/pulsar-jms-admin-ext/pom.xml
index d4754733..698c0711 100644
--- a/pulsar-jms-admin-ext/pom.xml
+++ b/pulsar-jms-admin-ext/pom.xml
@@ -66,7 +66,7 @@
       <plugin>
         <groupId>org.apache.nifi</groupId>
         <artifactId>nifi-nar-maven-plugin</artifactId>
-        <version>1.3.2</version>
+        <version>1.5.1</version>
         <extensions>true</extensions>
         <configuration>
           <finalName>pulsar-jms-admin-${project.version}</finalName>
diff --git a/pulsar-jms-filters/pom.xml b/pulsar-jms-filters/pom.xml
index 37e22a58..e8e86c98 100644
--- a/pulsar-jms-filters/pom.xml
+++ b/pulsar-jms-filters/pom.xml
@@ -85,7 +85,7 @@
       <plugin>
         <groupId>org.apache.nifi</groupId>
         <artifactId>nifi-nar-maven-plugin</artifactId>
-        <version>1.3.2</version>
+        <version>1.5.1</version>
         <extensions>true</extensions>
         <configuration>
           <finalName>pulsar-jms-${project.version}</finalName>
diff --git a/pulsar-jms-integration-tests/pom.xml b/pulsar-jms-integration-tests/pom.xml
index 798d61a8..747bb627 100644
--- a/pulsar-jms-integration-tests/pom.xml
+++ b/pulsar-jms-integration-tests/pom.xml
@@ -103,8 +103,8 @@
             <configuration>
               <target>
                 <echo>copy filters</echo>
-                <mkdir dir="${project.build.outputDirectory}/filters" />
-                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar" />
+                <mkdir dir="${project.build.outputDirectory}/filters"/>
+                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar"/>
               </target>
             </configuration>
           </execution>
diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index da7d2dda..fc87d02a 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -69,7 +69,7 @@
       <plugin>
         <groupId>org.apache.nifi</groupId>
         <artifactId>nifi-nar-maven-plugin</artifactId>
-        <version>1.3.2</version>
+        <version>1.5.1</version>
         <extensions>true</extensions>
         <configuration>
           <finalName>pulsar-jms-tracing-${project.version}</finalName>
diff --git a/pulsar-jms/pom.xml b/pulsar-jms/pom.xml
index b715013d..adc3d577 100644
--- a/pulsar-jms/pom.xml
+++ b/pulsar-jms/pom.xml
@@ -128,10 +128,10 @@
             <configuration>
               <target>
                 <echo>copy filters</echo>
-                <mkdir dir="${project.build.outputDirectory}/filters" />
-                <mkdir dir="${project.build.outputDirectory}/interceptors" />
-                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar" />
-                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/interceptors/jms-filter.nar" />
+                <mkdir dir="${project.build.outputDirectory}/filters"/>
+                <mkdir dir="${project.build.outputDirectory}/interceptors"/>
+                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/filters/jms-filter.nar"/>
+                <copy verbose="true" file="${basedir}/../pulsar-jms-filters/target/pulsar-jms-${project.version}-nar.nar" tofile="${project.build.outputDirectory}/interceptors/jms-filter.nar"/>
               </target>
             </configuration>
           </execution>

From 7c8750336fe46ebc7e5e89ac84c2e4f25c3ebdcb Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 16:31:24 +0200
Subject: [PATCH 12/29] fix build

---
 pulsar-jms-tracing/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index fc87d02a..4c8ff528 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <artifactId>pulsar-jms-parent</artifactId>
     <groupId>com.datastax.oss</groupId>
-    <version>4.1.1-alpha-SNAPSHOT</version>
+    <version>4.1.2-alpha-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>pulsar-jms-tracing</artifactId>

From 58f11836c4f06f80f80f5a7ab003605bea0c28c8 Mon Sep 17 00:00:00 2001
From: Enrico Olivelli <enrico.olivelli@datastax.com>
Date: Tue, 16 Apr 2024 16:48:49 +0200
Subject: [PATCH 13/29] fix build

---
 pulsar-jms-tracing/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index 4c8ff528..ed2bf109 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <artifactId>pulsar-jms-parent</artifactId>
     <groupId>com.datastax.oss</groupId>
-    <version>4.1.2-alpha-SNAPSHOT</version>
+    <version>4.1.2-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>pulsar-jms-tracing</artifactId>

From cbe3a5b034e037deaa380f1002a3323c7ccb91c0 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 16 Apr 2024 09:21:31 -0700
Subject: [PATCH 14/29] limit command output, separate event reason option for
 commands

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java |   7 +-
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 143 ++++++++++--------
 2 files changed, 86 insertions(+), 64 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 7606563f..a4c44f98 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -62,6 +62,7 @@ public class BrokerTracing implements BrokerInterceptor {
 
   public enum EventReasons {
     ADMINISTRATIVE,
+    COMMANDS,
     MESSAGE,
     TRANSACTION,
     SERVLET,
@@ -269,7 +270,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
   }
 
   public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
 
     if (traceLevel == TraceLevel.NONE) return;
 
@@ -278,7 +279,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
 
     if (command.hasType()) {
       traceDetails.put("type", command.getType().name());
-      traceDetails.put("command", getCommandDetails(traceLevel, command));
+      if (traceLevel != TraceLevel.MINIMAL) {
+        traceDetails.put("command", getCommandDetails(traceLevel, command));
+      }
     } else {
       traceDetails.put("type", "unknown/null");
     }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index cc6ba6de..86334ad6 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -18,12 +18,14 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.collect.Sets;
 import io.netty.buffer.ByteBuf;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -109,181 +111,180 @@ private static void populateCommandDetails(
     }
 
     // trace all params otherwise
-
     switch (command.getType()) {
       case CONNECT:
-        populateByReflection(command.getConnect(), traceDetails);
+        populateByReflection(level, command.getConnect(), traceDetails);
         break;
       case CONNECTED:
-        populateByReflection(command.getConnected(), traceDetails);
+        populateByReflection(level, command.getConnected(), traceDetails);
         break;
       case SUBSCRIBE:
-        populateByReflection(command.getSubscribe(), traceDetails);
+        populateByReflection(level, command.getSubscribe(), traceDetails);
         break;
       case PRODUCER:
-        populateByReflection(command.getProducer(), traceDetails);
+        populateByReflection(level, command.getProducer(), traceDetails);
         break;
       case SEND:
-        populateByReflection(command.getSend(), traceDetails);
+        populateByReflection(level, command.getSend(), traceDetails);
         break;
       case SEND_RECEIPT:
-        populateByReflection(command.getSendReceipt(), traceDetails);
+        populateByReflection(level, command.getSendReceipt(), traceDetails);
         break;
       case SEND_ERROR:
-        populateByReflection(command.getSendError(), traceDetails);
+        populateByReflection(level, command.getSendError(), traceDetails);
         break;
       case MESSAGE:
-        populateByReflection(command.getMessage(), traceDetails);
+        populateByReflection(level, command.getMessage(), traceDetails);
         break;
       case ACK:
-        populateByReflection(command.getAck(), traceDetails);
+        populateByReflection(level, command.getAck(), traceDetails);
         break;
       case FLOW:
-        populateByReflection(command.getFlow(), traceDetails);
+        populateByReflection(level, command.getFlow(), traceDetails);
         break;
       case UNSUBSCRIBE:
-        populateByReflection(command.getUnsubscribe(), traceDetails);
+        populateByReflection(level, command.getUnsubscribe(), traceDetails);
         break;
       case SUCCESS:
-        populateByReflection(command.getSuccess(), traceDetails);
+        populateByReflection(level, command.getSuccess(), traceDetails);
         break;
       case ERROR:
-        populateByReflection(command.getError(), traceDetails);
+        populateByReflection(level, command.getError(), traceDetails);
         break;
       case CLOSE_PRODUCER:
-        populateByReflection(command.getCloseProducer(), traceDetails);
+        populateByReflection(level, command.getCloseProducer(), traceDetails);
         break;
       case CLOSE_CONSUMER:
-        populateByReflection(command.getCloseConsumer(), traceDetails);
+        populateByReflection(level, command.getCloseConsumer(), traceDetails);
         break;
       case PRODUCER_SUCCESS:
-        populateByReflection(command.getProducerSuccess(), traceDetails);
+        populateByReflection(level, command.getProducerSuccess(), traceDetails);
         break;
       case PING:
-        populateByReflection(command.getPing(), traceDetails);
+        populateByReflection(level, command.getPing(), traceDetails);
         break;
       case PONG:
-        populateByReflection(command.getPong(), traceDetails);
+        populateByReflection(level, command.getPong(), traceDetails);
         break;
       case REDELIVER_UNACKNOWLEDGED_MESSAGES:
-        populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
+        populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails);
         break;
       case PARTITIONED_METADATA:
-        populateByReflection(command.getPartitionMetadata(), traceDetails);
+        populateByReflection(level, command.getPartitionMetadata(), traceDetails);
         break;
       case PARTITIONED_METADATA_RESPONSE:
-        populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
+        populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails);
         break;
       case LOOKUP:
-        populateByReflection(command.getLookupTopic(), traceDetails);
+        populateByReflection(level, command.getLookupTopic(), traceDetails);
         break;
       case LOOKUP_RESPONSE:
-        populateByReflection(command.getLookupTopicResponse(), traceDetails);
+        populateByReflection(level, command.getLookupTopicResponse(), traceDetails);
         break;
       case CONSUMER_STATS:
-        populateByReflection(command.getConsumerStats(), traceDetails);
+        populateByReflection(level, command.getConsumerStats(), traceDetails);
         break;
       case CONSUMER_STATS_RESPONSE:
-        populateByReflection(command.getConsumerStatsResponse(), traceDetails);
+        populateByReflection(level, command.getConsumerStatsResponse(), traceDetails);
         break;
       case REACHED_END_OF_TOPIC:
-        populateByReflection(command.getReachedEndOfTopic(), traceDetails);
+        populateByReflection(level, command.getReachedEndOfTopic(), traceDetails);
         break;
       case SEEK:
-        populateByReflection(command.getSeek(), traceDetails);
+        populateByReflection(level, command.getSeek(), traceDetails);
         break;
       case GET_LAST_MESSAGE_ID:
-        populateByReflection(command.getGetLastMessageId(), traceDetails);
+        populateByReflection(level, command.getGetLastMessageId(), traceDetails);
         break;
       case GET_LAST_MESSAGE_ID_RESPONSE:
-        populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
+        populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails);
         break;
       case ACTIVE_CONSUMER_CHANGE:
-        populateByReflection(command.getActiveConsumerChange(), traceDetails);
+        populateByReflection(level, command.getActiveConsumerChange(), traceDetails);
         break;
       case GET_TOPICS_OF_NAMESPACE:
-        populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
+        populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails);
         break;
       case GET_TOPICS_OF_NAMESPACE_RESPONSE:
-        populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
+        populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails);
         break;
       case GET_SCHEMA:
-        populateByReflection(command.getGetSchema(), traceDetails);
+        populateByReflection(level, command.getGetSchema(), traceDetails);
         break;
       case GET_SCHEMA_RESPONSE:
-        populateByReflection(command.getGetSchemaResponse(), traceDetails);
+        populateByReflection(level, command.getGetSchemaResponse(), traceDetails);
         break;
       case AUTH_CHALLENGE:
-        populateByReflection(command.getAuthChallenge(), traceDetails);
+        populateByReflection(level, command.getAuthChallenge(), traceDetails);
         break;
       case AUTH_RESPONSE:
-        populateByReflection(command.getAuthResponse(), traceDetails);
+        populateByReflection(level, command.getAuthResponse(), traceDetails);
         break;
       case ACK_RESPONSE:
-        populateByReflection(command.getAckResponse(), traceDetails);
+        populateByReflection(level, command.getAckResponse(), traceDetails);
         break;
       case GET_OR_CREATE_SCHEMA:
-        populateByReflection(command.getGetOrCreateSchema(), traceDetails);
+        populateByReflection(level, command.getGetOrCreateSchema(), traceDetails);
         break;
       case GET_OR_CREATE_SCHEMA_RESPONSE:
-        populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
+        populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails);
         break;
       case NEW_TXN:
-        populateByReflection(command.getNewTxn(), traceDetails);
+        populateByReflection(level, command.getNewTxn(), traceDetails);
         break;
       case NEW_TXN_RESPONSE:
-        populateByReflection(command.getNewTxnResponse(), traceDetails);
+        populateByReflection(level, command.getNewTxnResponse(), traceDetails);
         break;
       case ADD_PARTITION_TO_TXN:
-        populateByReflection(command.getAddPartitionToTxn(), traceDetails);
+        populateByReflection(level, command.getAddPartitionToTxn(), traceDetails);
         break;
       case ADD_PARTITION_TO_TXN_RESPONSE:
-        populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
+        populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails);
         break;
       case ADD_SUBSCRIPTION_TO_TXN:
-        populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
+        populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails);
         break;
       case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
-        populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
+        populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails);
         break;
       case END_TXN:
-        populateByReflection(command.getEndTxn(), traceDetails);
+        populateByReflection(level, command.getEndTxn(), traceDetails);
         break;
       case END_TXN_RESPONSE:
-        populateByReflection(command.getEndTxnResponse(), traceDetails);
+        populateByReflection(level, command.getEndTxnResponse(), traceDetails);
         break;
       case END_TXN_ON_PARTITION:
-        populateByReflection(command.getEndTxnOnPartition(), traceDetails);
+        populateByReflection(level, command.getEndTxnOnPartition(), traceDetails);
         break;
       case END_TXN_ON_PARTITION_RESPONSE:
-        populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
+        populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails);
         break;
       case END_TXN_ON_SUBSCRIPTION:
-        populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
+        populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails);
         break;
       case END_TXN_ON_SUBSCRIPTION_RESPONSE:
-        populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
+        populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails);
         break;
       case TC_CLIENT_CONNECT_REQUEST:
-        populateByReflection(command.getTcClientConnectRequest(), traceDetails);
+        populateByReflection(level, command.getTcClientConnectRequest(), traceDetails);
         break;
       case TC_CLIENT_CONNECT_RESPONSE:
-        populateByReflection(command.getTcClientConnectResponse(), traceDetails);
+        populateByReflection(level, command.getTcClientConnectResponse(), traceDetails);
         break;
       case WATCH_TOPIC_LIST:
-        populateByReflection(command.getWatchTopicList(), traceDetails);
+        populateByReflection(level, command.getWatchTopicList(), traceDetails);
         break;
       case WATCH_TOPIC_LIST_SUCCESS:
-        populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
+        populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails);
         break;
       case WATCH_TOPIC_UPDATE:
-        populateByReflection(command.getWatchTopicUpdate(), traceDetails);
+        populateByReflection(level, command.getWatchTopicUpdate(), traceDetails);
         break;
       case WATCH_TOPIC_LIST_CLOSE:
-        populateByReflection(command.getWatchTopicListClose(), traceDetails);
+        populateByReflection(level, command.getWatchTopicListClose(), traceDetails);
         break;
       case TOPIC_MIGRATED:
-        populateByReflection(command.getTopicMigrated(), traceDetails);
+        populateByReflection(level, command.getTopicMigrated(), traceDetails);
         break;
       default:
         log.error("Unknown command type: {}", command.getType());
@@ -291,7 +292,18 @@ private static void populateCommandDetails(
     }
   }
 
-  private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
+  private static final Set<String> fullTraceFields =
+      Sets.newHashSet(
+          "authdata",
+          "authmethod",
+          "authmethodname",
+          "originalauthdata",
+          "orginalauthmethod",
+          "originalprincipal",
+          "schema");
+
+  private static void populateByReflection(
+      TraceLevel level, Object command, Map<String, Object> traceDetails) {
     if (command == null) {
       return;
     }
@@ -302,7 +314,14 @@ private static void populateByReflection(Object command, Map<String, Object> tra
     Method[] allMethods = command.getClass().getMethods();
 
     Arrays.stream(allMethods)
-        .filter(method -> method.getName().startsWith("has"))
+        .filter(
+            method -> {
+              if (!method.getName().startsWith("has")) {
+                return false;
+              }
+              String fieldName = method.getName().substring(3);
+              return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase());
+            })
         .filter(
             method -> {
               try {
@@ -354,7 +373,7 @@ private static void populateByReflection(Object command, Map<String, Object> tra
                     .getCanonicalName()
                     .contains("org.apache.pulsar.common.api.proto")) {
                   Map<String, Object> details = new TreeMap<>();
-                  populateByReflection(value, details);
+                  populateByReflection(level, value, details);
                   traceDetails.put(fieldName, details);
                 } else {
                   traceDetails.put(fieldName, value);

From 7e480aadcf33b17a29bb1631a4c98e70e0f0c5c5 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 12:04:44 -0700
Subject: [PATCH 15/29] more configuration parameters to reduce verbosity

---
 .../oss/pulsar/jms/selectors/JMSFilter.java   |   3 +-
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 116 +++++++++++++-----
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  52 ++++----
 .../pulsar/jms/tracing/TracingUtilsTest.java  |  14 ++-
 4 files changed, 126 insertions(+), 59 deletions(-)

diff --git a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
index dc97f9f5..f8b22834 100644
--- a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
+++ b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java
@@ -68,7 +68,8 @@ public class JMSFilter implements EntryFilter {
         try {
           CollectorRegistry.defaultRegistry.register(filterProcessingTime);
         } catch (IllegalArgumentException alreadyRegistered) {
-          // this happens in Pulsar 2.10, because each JMSFilter is created in a different classloader
+          // this happens in Pulsar 2.10, because each JMSFilter is created in a different
+          // classloader
           // so the metricRegistered flag doesn't help
           log.debug("JMSFilter metrics already registered");
         }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index a4c44f98..7a331e69 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -34,6 +34,7 @@
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutionException;
@@ -72,6 +73,10 @@ public enum EventReasons {
 
   private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
   private TraceLevel traceLevel = defaultTraceLevel;
+  private int maxBinaryDataLength = 256;
+  private boolean traceSystemTopics = false;
+  private boolean traceSchema = false;
+  private boolean reduceLevelForNestedComponents = true;
 
   private static Set<EventReasons> loadEnabledEvents(
       PulsarService pulsarService, Set<EventReasons> enabledEvents) {
@@ -136,7 +141,7 @@ public TraceLevel load(Subscription sub) {
                 public TraceLevel load(Producer producer) {
                   if (producer.getMetadata() == null
                       || !producer.getMetadata().containsKey("trace")) {
-                    return TraceLevel.NONE;
+                    return TraceLevel.FULL;
                   }
                   try {
                     return TraceLevel.valueOf(
@@ -146,7 +151,7 @@ public TraceLevel load(Producer producer) {
                         "Invalid tracing level: {}. Setting to NONE for producer {}",
                         producer.getMetadata().get("trace"),
                         producer);
-                    return TraceLevel.NONE;
+                    return TraceLevel.FULL;
                   }
                 }
               });
@@ -156,6 +161,21 @@ public void initialize(PulsarService pulsarService) {
 
     loadEnabledEvents(pulsarService, jmsTracingEventList);
     traceLevel = getTraceLevel(pulsarService);
+
+    Properties props = pulsarService.getConfiguration().getProperties();
+    if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
+      maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
+    }
+    if (props.containsKey("jmsTracingTraceSystemTopics")) {
+      traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
+    }
+    if (props.containsKey("jmsTracingTraceSchema")) {
+      traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
+    }
+    if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) {
+      reduceLevelForNestedComponents =
+          Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
+    }
   }
 
   @Override
@@ -163,14 +183,17 @@ public void close() {
     log.info("Closing BrokerTracing");
   }
 
-  private static TraceLevel getTracingLevel(Consumer consumer) {
+  private TraceLevel getTracingLevel(Consumer consumer) {
     if (consumer == null) return TraceLevel.NONE;
+
     return getTracingLevel(consumer.getSubscription());
   }
 
-  private static TraceLevel getTracingLevel(Subscription sub) {
+  private TraceLevel getTracingLevel(Subscription sub) {
     if (sub == null) return TraceLevel.NONE;
 
+    if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE;
+
     try {
       return traceLevelForSubscription.get(sub);
     } catch (ExecutionException e) {
@@ -179,9 +202,11 @@ private static TraceLevel getTracingLevel(Subscription sub) {
     }
   }
 
-  private static TraceLevel getTracingLevel(Producer producer) {
+  private TraceLevel getTracingLevel(Producer producer) {
     if (producer == null) return TraceLevel.NONE;
 
+    if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE;
+
     try {
       return traceLevelForProducer.get(producer);
     } catch (ExecutionException e) {
@@ -190,6 +215,13 @@ private static TraceLevel getTracingLevel(Producer producer) {
     }
   }
 
+  private TraceLevel getTraceLevelForComponent(TraceLevel current) {
+    if (current == TraceLevel.NONE) return TraceLevel.NONE;
+    if (reduceLevelForNestedComponents) return TraceLevel.BASIC;
+
+    return current;
+  }
+
   //    private boolean needToTraceTopic(Topic topic) {
   //        if (topic == null) return false;
   //
@@ -218,8 +250,8 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+    traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
     trace("Producer created", traceDetails);
@@ -232,8 +264,8 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+    traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
     trace("Producer closed", traceDetails);
@@ -246,7 +278,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
     traceDetails.put("consumer", getConsumerDetails(level, consumer));
     traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
@@ -275,7 +307,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
     if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
 
     if (command.hasType()) {
       traceDetails.put("type", command.getType().name());
@@ -316,9 +348,10 @@ public void beforeSendMessage(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("subscription", getSubscriptionDetails(level, subscription));
-    traceDetails.put("consumer", getConsumerDetails(level, consumer));
-    traceDetails.put("entry", getEntryDetails(level, entry));
+    traceDetails.put(
+        "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription));
+    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+    traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength));
     traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
 
     trace("Before sending message", traceDetails);
@@ -333,9 +366,10 @@ public void onMessagePublish(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put(
+        "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
-    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
 
     trace("Message publish", traceDetails);
   }
@@ -353,11 +387,11 @@ public void messageProduced(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("producer", getProducerDetails(level, producer));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+    traceDetails.put(
+        "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
-    traceDetails.put("ledgerId", ledgerId);
-    traceDetails.put("entryId", entryId);
+    traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
 
     trace("Message produced", traceDetails);
@@ -371,12 +405,13 @@ public void messageDispatched(
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("consumer", getConsumerDetails(level, consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
-    traceDetails.put("ledgerId", ledgerId);
-    traceDetails.put("entryId", entryId);
-    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails);
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+    traceDetails.put(
+        "subscription",
+        getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+    traceDetails.put("messageId", ledgerId + ":" + entryId);
+    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
 
     trace("After dispatching message", traceDetails);
   }
@@ -388,13 +423,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (level == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("consumer", getConsumerDetails(level, consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
+    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
+    traceDetails.put(
+        "subscription",
+        getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
 
     Map<String, Object> ackDetails = new TreeMap<>();
-    ackDetails.put("type", ackCmd.getAckType().name());
-    ackDetails.put("consumerId", ackCmd.getConsumerId());
+    if (ackCmd.hasAckType()) {
+      ackDetails.put("type", ackCmd.getAckType().name());
+    } else {
+      ackDetails.put("type", "NOT SET");
+    }
+    if (ackCmd.hasConsumerId()) {
+      ackDetails.put("consumerId", ackCmd.getConsumerId());
+    } else {
+      ackDetails.put("consumerId", "NOT SET");
+    }
+    ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
     ackDetails.put(
         "messageIds",
         ackCmd
@@ -403,6 +449,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
             .map(x -> x.getLedgerId() + ":" + x.getEntryId())
             .collect(Collectors.toList()));
 
+    if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) {
+      ackDetails.put(
+          "txnID", "(" + ackCmd.getTxnidMostBits() + "," + ackCmd.getTxnidLeastBits() + ")");
+    }
+    if (ackCmd.hasRequestId()) {
+      ackDetails.put("requestId", ackCmd.getRequestId());
+    }
+
     traceDetails.put("ack", ackDetails);
 
     trace("Message acked", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 86334ad6..87c3ca13 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -39,6 +39,7 @@
 import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 
 @Slf4j
 public class TracingUtils {
@@ -57,8 +58,6 @@ public interface Tracer {
           .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
           .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
 
-  public static final int MAX_DATA_LENGTH = 1024;
-
   public enum TraceLevel {
     NONE,
     MINIMAL,
@@ -522,18 +521,19 @@ private static void populateConsumerDetails(
     }
   }
 
-  public static Map<String, Object> getProducerDetails(TraceLevel level, Producer producer) {
+  public static Map<String, Object> getProducerDetails(
+      TraceLevel level, Producer producer, boolean traceSchema) {
     if (producer == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateProducerDetails(level, producer, details);
+    populateProducerDetails(level, producer, details, traceSchema);
     return details;
   }
 
   private static void populateProducerDetails(
-      TraceLevel level, Producer producer, Map<String, Object> traceDetails) {
+      TraceLevel level, Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
     if (producer == null) {
       return;
     }
@@ -551,16 +551,25 @@ private static void populateProducerDetails(
         }
         break;
       case BASIC:
-        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails);
+        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema);
 
         traceDetails.put("clientAddress", producer.getClientAddress());
         break;
       case FULL:
-        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails);
+        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema);
 
         traceDetails.put("metadata", producer.getMetadata());
-        if (producer.getSchemaVersion() != null) {
-          traceDetails.put("schemaVersion", producer.getSchemaVersion().toString());
+
+        if (traceSchema && producer.getSchemaVersion() != null) {
+          final String schemaVersion;
+          if (producer.getSchemaVersion() == SchemaVersion.Empty) {
+            schemaVersion = "Empty";
+          } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
+            schemaVersion = "Latest";
+          } else {
+            schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
+          }
+          traceDetails.put("schemaVersion", schemaVersion);
         }
         traceDetails.put("remoteCluster", producer.getRemoteCluster());
         break;
@@ -636,36 +645,36 @@ private static void populateMessageMetadataDetails(
     }
   }
 
-  public static Map<String, Object> getEntryDetails(TraceLevel level, Entry entry) {
+  public static Map<String, Object> getEntryDetails(
+      TraceLevel level, Entry entry, int maxBinaryDataLength) {
     if (entry == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateEntryDetails(level, entry, details);
+    populateEntryDetails(level, entry, details, maxBinaryDataLength);
     return details;
   }
 
   private static void populateEntryDetails(
-      TraceLevel level, Entry entry, Map<String, Object> traceDetails) {
+      TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
     if (entry == null) {
       return;
     }
 
     switch (level) {
       case MINIMAL:
-        traceDetails.put("ledgerId", entry.getLedgerId());
-        traceDetails.put("entryId", entry.getEntryId());
+        traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
         break;
       case BASIC:
-        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails);
+        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength);
 
         traceDetails.put("length", entry.getLength());
         break;
       case FULL:
-        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails);
+        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength);
 
-        traceByteBuf("data", entry.getDataBuffer(), traceDetails);
+        traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
         break;
       case NONE:
         break;
@@ -699,14 +708,15 @@ private static void populatePublishContext(
     traceDetails.put("sequenceId", publishContext.getSequenceId());
   }
 
-  public static void traceByteBuf(String key, ByteBuf buf, Map<String, Object> traceDetails) {
-    if (buf == null) return;
+  public static void traceByteBuf(
+      String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+    if (buf == null || maxBinaryDataLength <= 0) return;
 
-    if (buf.readableBytes() < MAX_DATA_LENGTH) {
+    if (buf.readableBytes() < maxBinaryDataLength) {
       traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
     } else {
       traceDetails.put(
-          key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer()));
+          key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()));
     }
   }
 }
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index fa148cc8..220fee87 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -92,27 +92,29 @@ void traceTest() {
   void traceByteBufTest() {
     Map<String, Object> traceDetails = new TreeMap<>();
 
-    traceByteBuf("key", null, traceDetails);
+    int maxBinaryDataLength = 1024;
+
+    traceByteBuf("key", null, traceDetails, maxBinaryDataLength);
     assertEquals(0, traceDetails.size());
 
     ByteBuf small = Unpooled.buffer(20);
     for (int i = 0; i < 20; i++) {
       small.writeByte(i);
     }
-    traceByteBuf("key", small, traceDetails);
+    traceByteBuf("key", small, traceDetails, maxBinaryDataLength);
     assertEquals(1, traceDetails.size());
     assertEquals(42, ((String) traceDetails.get("key")).length());
     assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
 
-    ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100);
-    for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) {
+    ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100);
+    for (int i = 0; i < maxBinaryDataLength + 100; i++) {
       big.writeByte(i);
     }
 
     traceDetails.clear();
-    traceByteBuf("key", big, traceDetails);
+    traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
     assertEquals(1, traceDetails.size());
-    assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length());
+    assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length());
     assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
   }
 }

From 0a42c1f50a818d74493e2fd6dc9c55b451847a11 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 14:43:02 -0700
Subject: [PATCH 16/29] updated version to match master / merged poms

---
 pulsar-jms-tracing/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml
index ed2bf109..19b588df 100644
--- a/pulsar-jms-tracing/pom.xml
+++ b/pulsar-jms-tracing/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <artifactId>pulsar-jms-parent</artifactId>
     <groupId>com.datastax.oss</groupId>
-    <version>4.1.2-SNAPSHOT</version>
+    <version>4.1.3-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>pulsar-jms-tracing</artifactId>

From 9c1ad66b0f8fc094876e4855a2c8f51ff98bf7c2 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 29 Apr 2024 15:58:06 -0700
Subject: [PATCH 17/29] more message id details, when available

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java     | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 7a331e69..2ab6da2b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.CommandAck;
+import org.apache.pulsar.common.api.proto.MessageIdData;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.intercept.InterceptException;
 import org.jetbrains.annotations.NotNull;
@@ -393,7 +394,6 @@ public void messageProduced(
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
-
     trace("Message produced", traceDetails);
   }
 
@@ -446,7 +446,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
         ackCmd
             .getMessageIdsList()
             .stream()
-            .map(x -> x.getLedgerId() + ":" + x.getEntryId())
+            .map(BrokerTracing::formatMessageId)
             .collect(Collectors.toList()));
 
     if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) {
@@ -462,6 +462,17 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     trace("Message acked", traceDetails);
   }
 
+  @NotNull
+  private static String formatMessageId(MessageIdData x) {
+    String msgId = x.getLedgerId() + ":" + x.getEntryId();
+    if (x.hasBatchIndex()) {
+      msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
+    } else if (x.getAckSetsCount() > 0) {
+      msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
+    }
+    return msgId;
+  }
+
   /* ***************************
    **  Transaction events
    ******************************/

From a9d74f8a05e7272d60e9477759563cfee8dbb879 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 30 Apr 2024 15:44:43 -0700
Subject: [PATCH 18/29] Trace levels for producer/consumer can be set via topic
 properties; a bit more infor for auth is traced

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++-------
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  37 +++++-
 2 files changed, 112 insertions(+), 46 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 2ab6da2b..98aa617b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -52,11 +52,13 @@
 import org.apache.pulsar.broker.service.ServerCnx;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.CommandAck;
 import org.apache.pulsar.common.api.proto.MessageIdData;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.naming.TopicName;
 import org.jetbrains.annotations.NotNull;
 
 @Slf4j
@@ -75,6 +77,7 @@ public enum EventReasons {
   private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
   private TraceLevel traceLevel = defaultTraceLevel;
   private int maxBinaryDataLength = 256;
+  private int cacheTraceLevelsDurationSec = 10;
   private boolean traceSystemTopics = false;
   private boolean traceSchema = false;
   private boolean reduceLevelForNestedComponents = true;
@@ -111,18 +114,20 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
     }
   }
 
-  private static final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
+  private final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(10, TimeUnit.SECONDS)
+          .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
           .build(
               new CacheLoader<Subscription, TraceLevel>() {
                 public TraceLevel load(Subscription sub) {
                   Map<String, String> subProps = sub.getSubscriptionProperties();
-                  if (subProps == null || !subProps.containsKey("trace")) {
-                    return TraceLevel.NONE;
-                  }
-
                   try {
+                    if (subProps == null || !subProps.containsKey("trace")) {
+                      PulsarAdmin admin =
+                          sub.getTopic().getBrokerService().getPulsar().getAdminClient();
+                      return BrokerTracing.getTracingLevel(admin, sub.getTopic());
+                    }
+
                     return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
                   } catch (IllegalArgumentException e) {
                     log.warn(
@@ -130,33 +135,55 @@ public TraceLevel load(Subscription sub) {
                         subProps.get("trace"),
                         sub);
                     return TraceLevel.NONE;
+                  } catch (Throwable t) {
+                    log.error("Error getting tracing level", t);
+                    return TraceLevel.NONE;
                   }
                 }
               });
 
-  private static final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
+  private final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(10, TimeUnit.SECONDS)
+          .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
           .build(
               new CacheLoader<Producer, TraceLevel>() {
                 public TraceLevel load(Producer producer) {
-                  if (producer.getMetadata() == null
-                      || !producer.getMetadata().containsKey("trace")) {
-                    return TraceLevel.FULL;
-                  }
                   try {
-                    return TraceLevel.valueOf(
-                        producer.getMetadata().get("trace").trim().toUpperCase());
-                  } catch (IllegalArgumentException e) {
-                    log.warn(
-                        "Invalid tracing level: {}. Setting to NONE for producer {}",
-                        producer.getMetadata().get("trace"),
-                        producer);
-                    return TraceLevel.FULL;
+                    PulsarAdmin admin =
+                        producer.getCnx().getBrokerService().getPulsar().getAdminClient();
+                    Topic topic = producer.getTopic();
+
+                    return BrokerTracing.getTracingLevel(admin, topic);
+                  } catch (Throwable t) {
+                    log.error("Error getting tracing level", t);
+                    return TraceLevel.NONE;
                   }
                 }
               });
 
+  @NotNull
+  private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) {
+    Map<String, String> props = null;
+    try {
+      props =
+          admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName());
+
+      if (props == null || !props.containsKey("trace")) {
+        return TraceLevel.NONE;
+      }
+
+      return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+    } catch (IllegalArgumentException e) {
+      log.warn(
+          "Invalid tracing level: {}. Setting to NONE",
+          props == null ? "no props" : props.get("trace"));
+      return TraceLevel.NONE;
+    } catch (Throwable t) {
+      log.error("Error getting tracing level", t);
+      return TraceLevel.NONE;
+    }
+  }
+
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
 
@@ -177,6 +204,15 @@ public void initialize(PulsarService pulsarService) {
       reduceLevelForNestedComponents =
           Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
     }
+    if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
+      cacheTraceLevelsDurationSec =
+          Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec"));
+      if (cacheTraceLevelsDurationSec <= 0) {
+        log.warn(
+            "Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10);
+        cacheTraceLevelsDurationSec = 10;
+      }
+    }
   }
 
   @Override
@@ -223,13 +259,6 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) {
     return current;
   }
 
-  //    private boolean needToTraceTopic(Topic topic) {
-  //        if (topic == null) return false;
-  //
-  //        // todo: how to read topic props
-  //        return true;
-  //    }
-
   /* ***************************
    **  Administrative events
    ******************************/
@@ -246,13 +275,13 @@ public void onConnectionCreated(ServerCnx cnx) {
 
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    TraceLevel level = getTracingLevel(producer);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+    traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
     trace("Producer created", traceDetails);
@@ -260,13 +289,13 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    TraceLevel level = getTracingLevel(producer);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put("producer", getProducerDetails(level, producer, traceSchema));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+    traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
     trace("Producer closed", traceDetails);
@@ -274,14 +303,15 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
 
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    TraceLevel level = getTracingLevel(consumer);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put("consumer", getConsumerDetails(level, consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+    traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
+    traceDetails.put(
+        "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
     trace("Consumer created", traceDetails);
@@ -289,14 +319,15 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    TraceLevel level = getTracingLevel(consumer);
-    if (level == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.NONE) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(level, cnx));
-    traceDetails.put("consumer", getConsumerDetails(level, consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+    traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
+    traceDetails.put(
+        "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
     trace("Consumer closed", traceDetails);
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 87c3ca13..4e6311df 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -31,6 +31,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.commons.codec.binary.Hex;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.Producer;
 import org.apache.pulsar.broker.service.ServerCnx;
@@ -402,6 +403,7 @@ private static void populateConnectionDetails(
     switch (level) {
       case MINIMAL:
         traceDetails.put("clientAddress", cnx.clientAddress().toString());
+        traceDetails.put("authRole", cnx.getAuthRole());
         break;
       case BASIC:
         populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
@@ -412,13 +414,17 @@ private static void populateConnectionDetails(
       case FULL:
         populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
 
-        traceDetails.put("authRole", cnx.getAuthRole());
         traceDetails.put("authMethod", cnx.getAuthMethod());
         traceDetails.put(
             "authMethodName",
             cnx.getAuthenticationProvider() == null
                 ? "no provider"
                 : cnx.getAuthenticationProvider().getAuthMethodName());
+
+        AuthenticationDataSource authData = cnx.getAuthenticationData();
+        if (authData != null) {
+          traceDetails.put("authData", getAuthDataDetails(authData));
+        }
         break;
       case NONE:
         break;
@@ -428,6 +434,35 @@ private static void populateConnectionDetails(
     }
   }
 
+  private static Object getAuthDataDetails(AuthenticationDataSource authData) {
+    if (authData == null) {
+      return null;
+    }
+
+    Map<String, Object> details = new TreeMap<>();
+    populateAuthDataDetails(authData, details);
+    return details;
+  }
+
+  private static void populateAuthDataDetails(
+      AuthenticationDataSource authData, Map<String, Object> details) {
+    if (authData == null) {
+      return;
+    }
+
+    details.put("peerAddress", authData.getPeerAddress());
+    details.put("commandData", authData.getCommandData());
+    details.put("httpAuthType", authData.getHttpAuthType());
+    details.put("subscription", authData.getSubscription());
+    if (authData.getTlsCertificates() != null) {
+      details.put(
+          "tlsCertificates",
+          Arrays.stream(authData.getTlsCertificates())
+              .map(Object::toString)
+              .collect(Collectors.toList()));
+    }
+  }
+
   public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
     if (sub == null) {
       return null;

From e910fa6170de5cb2a553b17218d6a61d3d6270a8 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 1 May 2024 12:40:06 -0700
Subject: [PATCH 19/29] async refresh for caches

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 151 ++++++++++++++----
 1 file changed, 118 insertions(+), 33 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 98aa617b..8a77abee 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -30,6 +30,8 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
 import java.util.HashSet;
@@ -37,6 +39,7 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -116,74 +119,156 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
 
   private final LoadingCache<Subscription, TraceLevel> traceLevelForSubscription =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+          .maximumSize(10_000L)
+          .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+          .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+          .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
           .build(
               new CacheLoader<Subscription, TraceLevel>() {
                 public TraceLevel load(Subscription sub) {
-                  Map<String, String> subProps = sub.getSubscriptionProperties();
-                  try {
-                    if (subProps == null || !subProps.containsKey("trace")) {
-                      PulsarAdmin admin =
-                          sub.getTopic().getBrokerService().getPulsar().getAdminClient();
-                      return BrokerTracing.getTracingLevel(admin, sub.getTopic());
-                    }
-
-                    return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase());
-                  } catch (IllegalArgumentException e) {
-                    log.warn(
-                        "Invalid tracing level: {}. Setting to NONE for subscription {}",
-                        subProps.get("trace"),
-                        sub);
-                    return TraceLevel.NONE;
-                  } catch (Throwable t) {
-                    log.error("Error getting tracing level", t);
-                    return TraceLevel.NONE;
-                  }
+                  log.info("Loading trace level for subscription {}", sub);
+                  return BrokerTracing.readTraceLevelForSubscription(sub);
                 }
-              });
 
+                public ListenableFuture<TraceLevel> reload(Subscription sub, TraceLevel oldValue)
+                    throws Exception {
+                  SettableFuture<TraceLevel> future = SettableFuture.create();
+                  BrokerTracing.readTraceLevelForSubscriptionAsync(sub)
+                      .whenComplete(
+                          (level, ex) -> {
+                            if (ex != null) {
+                              future.setException(ex);
+                            } else {
+                              future.set(level);
+                            }
+                          });
+                  return future;
+                }
+              });
   private final LoadingCache<Producer, TraceLevel> traceLevelForProducer =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+          .maximumSize(10_000L)
+          .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+          .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
+          .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS)
           .build(
               new CacheLoader<Producer, TraceLevel>() {
                 public TraceLevel load(Producer producer) {
                   try {
+                    log.info("Loading trace level for producer {}", producer);
                     PulsarAdmin admin =
                         producer.getCnx().getBrokerService().getPulsar().getAdminClient();
                     Topic topic = producer.getTopic();
 
-                    return BrokerTracing.getTracingLevel(admin, topic);
+                    return BrokerTracing.readTraceLevelForTopic(admin, topic);
                   } catch (Throwable t) {
                     log.error("Error getting tracing level", t);
                     return TraceLevel.NONE;
                   }
                 }
+
+                public ListenableFuture<TraceLevel> reload(Producer producer, TraceLevel oldValue)
+                    throws Exception {
+                  SettableFuture<TraceLevel> future = SettableFuture.create();
+
+                  PulsarAdmin admin =
+                      producer.getCnx().getBrokerService().getPulsar().getAdminClient();
+                  Topic topic = producer.getTopic();
+
+                  BrokerTracing.readTraceLevelForTopicAsync(admin, topic)
+                      .whenComplete(
+                          (level, ex) -> {
+                            if (ex != null) {
+                              future.setException(ex);
+                            } else {
+                              future.set(level);
+                            }
+                          });
+
+                  return future;
+                }
               });
 
   @NotNull
-  private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) {
-    Map<String, String> props = null;
+  private static TraceLevel readTraceLevelForSubscription(Subscription sub) {
     try {
-      props =
-          admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName());
+      return readTraceLevelForSubscriptionAsync(sub).get();
+    } catch (InterruptedException | ExecutionException e) {
+      log.error("Interrupted while getting subscription tracing level for {}", sub, e);
+      Thread.currentThread().interrupt();
+      return TraceLevel.NONE;
+    } catch (Throwable t) {
+      log.error("Error getting subscription tracing level for {}", sub, t);
+      return TraceLevel.NONE;
+    }
+  }
 
-      if (props == null || !props.containsKey("trace")) {
-        return TraceLevel.NONE;
+  @NotNull
+  private static CompletableFuture<TraceLevel> readTraceLevelForSubscriptionAsync(
+      Subscription sub) {
+    Map<String, String> subProps = sub.getSubscriptionProperties();
+    try {
+      if (subProps == null || !subProps.containsKey("trace")) {
+        PulsarAdmin admin = sub.getTopic().getBrokerService().getPulsar().getAdminClient();
+        return BrokerTracing.readTraceLevelForTopicAsync(admin, sub.getTopic());
       }
 
-      return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+      return CompletableFuture.completedFuture(
+          TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase()));
     } catch (IllegalArgumentException e) {
       log.warn(
-          "Invalid tracing level: {}. Setting to NONE",
-          props == null ? "no props" : props.get("trace"));
+          "Invalid tracing level: {}. Setting to NONE for subscription {}",
+          subProps.get("trace"),
+          sub);
+      return CompletableFuture.completedFuture(TraceLevel.NONE);
+    } catch (Throwable t) {
+      log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t);
+      return CompletableFuture.completedFuture(TraceLevel.NONE);
+    }
+  }
+
+  @NotNull
+  private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic) {
+    try {
+      return readTraceLevelForTopicAsync(admin, topic).get();
+    } catch (InterruptedException | ExecutionException e) {
+      log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e);
+      Thread.currentThread().interrupt();
       return TraceLevel.NONE;
     } catch (Throwable t) {
-      log.error("Error getting tracing level", t);
+      log.error("Error getting tracing level for topic {}", topic.getName(), t);
       return TraceLevel.NONE;
     }
   }
 
+  @NotNull
+  private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
+      PulsarAdmin admin, Topic topic) {
+    CompletableFuture<Map<String, String>> propsFuture =
+        admin.topics().getPropertiesAsync(TopicName.get(topic.getName()).getPartitionedTopicName());
+    return propsFuture.handle(
+        (props, ex) -> {
+          if (ex != null) {
+            log.error("Error getting tracing level for topic {}", topic.getName(), ex);
+            return TraceLevel.NONE;
+          }
+
+          try {
+            if (props == null || !props.containsKey("trace")) {
+              return TraceLevel.NONE;
+            }
+
+            return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
+          } catch (IllegalArgumentException e) {
+            log.warn(
+                "Invalid tracing level for topic {}: {}. Setting to NONE",
+                topic.getName(),
+                props.get("trace"));
+            return TraceLevel.NONE;
+          }
+        });
+  }
+
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
 

From b1df8d8dea18e3fb1c40e036df2c2b45bbb6b4ef Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 16:04:38 -0700
Subject: [PATCH 20/29] part of the feedback addressed, only one level of
 onfo(still too verbose), better grabularaty for logger

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 207 ++++----
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 497 ++++++++----------
 .../pulsar/jms/tracing/TracingUtilsTest.java  |   6 +-
 3 files changed, 304 insertions(+), 406 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 8a77abee..68e34beb 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,6 +15,7 @@
  */
 package com.datastax.oss.pulsar.jms.tracing;
 
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
@@ -34,6 +35,7 @@
 import com.google.common.util.concurrent.SettableFuture;
 import io.netty.buffer.ByteBuf;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Properties;
@@ -67,15 +69,7 @@
 @Slf4j
 public class BrokerTracing implements BrokerInterceptor {
 
-  public enum EventReasons {
-    ADMINISTRATIVE,
-    COMMANDS,
-    MESSAGE,
-    TRANSACTION,
-    SERVLET,
-  }
-
-  private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC;
+  private static final TraceLevel defaultTraceLevel = TraceLevel.OFF;
 
   private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
   private TraceLevel traceLevel = defaultTraceLevel;
@@ -83,23 +77,26 @@ public enum EventReasons {
   private int cacheTraceLevelsDurationSec = 10;
   private boolean traceSystemTopics = false;
   private boolean traceSchema = false;
-  private boolean reduceLevelForNestedComponents = true;
 
-  private static Set<EventReasons> loadEnabledEvents(
+  private static void loadEnabledEvents(
       PulsarService pulsarService, Set<EventReasons> enabledEvents) {
-    String events =
-        pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", "");
-    log.debug("read jmsTracingEventList: {}", events);
-
-    for (String event : events.split(",")) {
-      try {
-        enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
-      } catch (IllegalArgumentException e) {
-        log.error("Invalid event: {}. Skipping", event);
+    Properties props = pulsarService.getConfiguration().getProperties();
+    if (props.contains("jmsTracingEventList")) {
+      String events = props.getProperty("jmsTracingEventList", "");
+      log.debug("read jmsTracingEventList: {}", events);
+
+      enabledEvents.clear();
+      for (String event : events.split(",")) {
+        try {
+          enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
+        } catch (IllegalArgumentException e) {
+          log.error("Invalid event: {}. Skipping", event);
+        }
       }
+    } else {
+      log.warn("jmsTracingEventList not set. Using all available.");
+      enabledEvents.addAll(Arrays.asList(EventReasons.values()));
     }
-
-    return enabledEvents;
   }
 
   @NotNull
@@ -163,7 +160,7 @@ public TraceLevel load(Producer producer) {
                     return BrokerTracing.readTraceLevelForTopic(admin, topic);
                   } catch (Throwable t) {
                     log.error("Error getting tracing level", t);
-                    return TraceLevel.NONE;
+                    return TraceLevel.OFF;
                   }
                 }
 
@@ -196,10 +193,10 @@ private static TraceLevel readTraceLevelForSubscription(Subscription sub) {
     } catch (InterruptedException | ExecutionException e) {
       log.error("Interrupted while getting subscription tracing level for {}", sub, e);
       Thread.currentThread().interrupt();
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     } catch (Throwable t) {
       log.error("Error getting subscription tracing level for {}", sub, t);
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     }
   }
 
@@ -220,10 +217,10 @@ private static CompletableFuture<TraceLevel> readTraceLevelForSubscriptionAsync(
           "Invalid tracing level: {}. Setting to NONE for subscription {}",
           subProps.get("trace"),
           sub);
-      return CompletableFuture.completedFuture(TraceLevel.NONE);
+      return CompletableFuture.completedFuture(TraceLevel.OFF);
     } catch (Throwable t) {
       log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t);
-      return CompletableFuture.completedFuture(TraceLevel.NONE);
+      return CompletableFuture.completedFuture(TraceLevel.OFF);
     }
   }
 
@@ -234,10 +231,10 @@ private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic)
     } catch (InterruptedException | ExecutionException e) {
       log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e);
       Thread.currentThread().interrupt();
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     } catch (Throwable t) {
       log.error("Error getting tracing level for topic {}", topic.getName(), t);
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     }
   }
 
@@ -250,12 +247,12 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
         (props, ex) -> {
           if (ex != null) {
             log.error("Error getting tracing level for topic {}", topic.getName(), ex);
-            return TraceLevel.NONE;
+            return TraceLevel.OFF;
           }
 
           try {
             if (props == null || !props.containsKey("trace")) {
-              return TraceLevel.NONE;
+              return TraceLevel.OFF;
             }
 
             return TraceLevel.valueOf(props.get("trace").trim().toUpperCase());
@@ -264,7 +261,7 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
                 "Invalid tracing level for topic {}: {}. Setting to NONE",
                 topic.getName(),
                 props.get("trace"));
-            return TraceLevel.NONE;
+            return TraceLevel.OFF;
           }
         });
   }
@@ -285,10 +282,6 @@ public void initialize(PulsarService pulsarService) {
     if (props.containsKey("jmsTracingTraceSchema")) {
       traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
     }
-    if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) {
-      reduceLevelForNestedComponents =
-          Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents"));
-    }
     if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
       cacheTraceLevelsDurationSec =
           Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec"));
@@ -306,44 +299,37 @@ public void close() {
   }
 
   private TraceLevel getTracingLevel(Consumer consumer) {
-    if (consumer == null) return TraceLevel.NONE;
+    if (consumer == null) return TraceLevel.OFF;
 
     return getTracingLevel(consumer.getSubscription());
   }
 
   private TraceLevel getTracingLevel(Subscription sub) {
-    if (sub == null) return TraceLevel.NONE;
+    if (sub == null) return TraceLevel.OFF;
 
-    if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE;
+    if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.OFF;
 
     try {
       return traceLevelForSubscription.get(sub);
     } catch (ExecutionException e) {
       log.error("Error getting tracing level", e);
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     }
   }
 
   private TraceLevel getTracingLevel(Producer producer) {
-    if (producer == null) return TraceLevel.NONE;
+    if (producer == null) return TraceLevel.OFF;
 
-    if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE;
+    if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.OFF;
 
     try {
       return traceLevelForProducer.get(producer);
     } catch (ExecutionException e) {
       log.error("Error getting tracing level", e);
-      return TraceLevel.NONE;
+      return TraceLevel.OFF;
     }
   }
 
-  private TraceLevel getTraceLevelForComponent(TraceLevel current) {
-    if (current == TraceLevel.NONE) return TraceLevel.NONE;
-    if (reduceLevelForNestedComponents) return TraceLevel.BASIC;
-
-    return current;
-  }
-
   /* ***************************
    **  Administrative events
    ******************************/
@@ -351,102 +337,98 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) {
   public void onConnectionCreated(ServerCnx cnx) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
-    trace("Connection created", traceDetails);
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
   }
 
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
     if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
-    traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
-    trace("Producer created", traceDetails);
+    trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
   }
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
     if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
-    traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
-    trace("Producer closed", traceDetails);
+    trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
   }
 
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
     if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
-    traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
-    traceDetails.put(
-        "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("consumer", getConsumerDetails(consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
-    trace("Consumer created", traceDetails);
+    trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
   }
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
     if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
-    traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer));
-    traceDetails.put(
-        "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("consumer", getConsumerDetails(consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
-    trace("Consumer closed", traceDetails);
+    trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
   }
 
   public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
     if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
 
     if (command.hasType()) {
       traceDetails.put("type", command.getType().name());
-      if (traceLevel != TraceLevel.MINIMAL) {
-        traceDetails.put("command", getCommandDetails(traceLevel, command));
-      }
+      traceDetails.put("command", getCommandDetails(command));
     } else {
       traceDetails.put("type", "unknown/null");
     }
 
-    trace("Pulsar command called", traceDetails);
+    trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails);
   }
 
   public void onConnectionClosed(ServerCnx cnx) {
     if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
 
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
 
-    trace("Connection closed", traceDetails);
+    trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
   }
 
   /* ***************************
@@ -462,16 +444,15 @@ public void beforeSendMessage(
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(subscription);
-    if (level == TraceLevel.NONE) return;
+    if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put(
-        "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription));
-    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
-    traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength));
-    traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata));
+    traceDetails.put("subscription", getSubscriptionDetails(subscription));
+    traceDetails.put("consumer", getConsumerDetails(consumer));
+    traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+    traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
 
-    trace("Before sending message", traceDetails);
+    trace(EventReasons.MESSAGE, "Before sending message", traceDetails);
   }
 
   public void onMessagePublish(
@@ -480,15 +461,14 @@ public void onMessagePublish(
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(producer);
-    if (level == TraceLevel.NONE) return;
+    if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put(
-        "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
+    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
 
-    trace("Message publish", traceDetails);
+    trace(EventReasons.MESSAGE, "Message publish", traceDetails);
   }
 
   public void messageProduced(
@@ -501,16 +481,15 @@ public void messageProduced(
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(producer);
-    if (level == TraceLevel.NONE) return;
+    if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put(
-        "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
-    trace("Message produced", traceDetails);
+    trace(EventReasons.MESSAGE, "Message produced", traceDetails);
   }
 
   public void messageDispatched(
@@ -518,32 +497,28 @@ public void messageDispatched(
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(consumer);
-    if (level == TraceLevel.NONE) return;
+    if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
-    traceDetails.put(
-        "subscription",
-        getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("consumer", getConsumerDetails(consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
 
-    trace("After dispatching message", traceDetails);
+    trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(consumer);
-    if (level == TraceLevel.NONE) return;
+    if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx));
-    traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer));
-    traceDetails.put(
-        "subscription",
-        getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription()));
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("consumer", getConsumerDetails(consumer));
+    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
 
     Map<String, Object> ackDetails = new TreeMap<>();
     if (ackCmd.hasAckType()) {
@@ -575,7 +550,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
 
     traceDetails.put("ack", ackDetails);
 
-    trace("Message acked", traceDetails);
+    trace(EventReasons.MESSAGE, "Message acked", traceDetails);
   }
 
   @NotNull
@@ -595,24 +570,24 @@ private static String formatMessageId(MessageIdData x) {
 
   public void txnOpened(long tcId, String txnID) {
     if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("tcId", tcId);
     traceDetails.put("txnID", txnID);
 
-    trace("Transaction opened", traceDetails);
+    trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails);
   }
 
   public void txnEnded(String txnID, long txnAction) {
     if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
-    if (traceLevel == TraceLevel.NONE) return;
+    if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("txnID", txnID);
     traceDetails.put("txnAction", txnAction);
 
-    trace("Transaction closed", traceDetails);
+    trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails);
   }
 
   /* ***************************
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 4e6311df..d995387a 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -23,6 +23,7 @@
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -44,15 +45,38 @@
 
 @Slf4j
 public class TracingUtils {
-  private static final org.slf4j.Logger traceLogger =
-      org.slf4j.LoggerFactory.getLogger("jms-tracing");
+
+  public enum EventReasons {
+    ADMINISTRATIVE,
+    COMMANDS,
+    MESSAGE,
+    TRANSACTION,
+    SERVLET,
+  }
 
   @FunctionalInterface
   public interface Tracer {
-    void trace(String message);
+    void trace(EventReasons reason, String message);
   }
 
-  public static final Tracer SLF4J_TRACER = traceLogger::info;
+  public static class Slf4jTracer implements Tracer {
+    private static final Map<EventReasons, org.slf4j.Logger> traceLoggers = new HashMap<>();
+
+    static {
+      for (EventReasons reason : EventReasons.values()) {
+        traceLoggers.put(
+            reason,
+            org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase()));
+      }
+    }
+
+    @Override
+    public void trace(EventReasons reason, String message) {
+      traceLoggers.get(reason).info(message);
+    }
+  }
+
+  public static final Tracer SLF4J_TRACER = new Slf4jTracer();
 
   private static final ObjectMapper mapper =
       new ObjectMapper()
@@ -60,24 +84,23 @@ public interface Tracer {
           .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
 
   public enum TraceLevel {
-    NONE,
-    MINIMAL,
-    BASIC,
-    FULL
+    OFF,
+    ON
   }
 
-  public static void trace(String message, Map<String, Object> traceDetails) {
-    trace(SLF4J_TRACER, message, traceDetails);
+  public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
+    trace(SLF4J_TRACER, reason, message, traceDetails);
   }
 
-  public static void trace(Tracer tracer, String message, Map<String, Object> traceDetails) {
+  public static void trace(
+      Tracer tracer, EventReasons reason, String message, Map<String, Object> traceDetails) {
     Map<String, Object> trace = new TreeMap<>();
     trace.put("eventType", message);
     trace.put("traceDetails", traceDetails);
 
     try {
       String loggableJsonString = mapper.writeValueAsString(trace);
-      tracer.trace(loggableJsonString);
+      tracer.trace(reason, loggableJsonString);
     } catch (JsonProcessingException e) {
       log.error(
           "Failed to serialize trace event type '{}' as json, traceDetails: {}",
@@ -87,24 +110,21 @@ public static void trace(Tracer tracer, String message, Map<String, Object> trac
     }
   }
 
-  public static Map<String, Object> getCommandDetails(TraceLevel level, BaseCommand command) {
+  public static Map<String, Object> getCommandDetails(BaseCommand command) {
     if (command == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateCommandDetails(level, command, details);
+    populateCommandDetails(command, details);
     return details;
   }
 
   private static void populateCommandDetails(
-      TraceLevel level, BaseCommand command, Map<String, Object> traceDetails) {
+      BaseCommand command, Map<String, Object> traceDetails) {
     if (command == null) {
       return;
     }
-    if (level == TraceLevel.NONE) {
-      return;
-    }
 
     if (!command.hasType()) {
       return;
@@ -113,178 +133,178 @@ private static void populateCommandDetails(
     // trace all params otherwise
     switch (command.getType()) {
       case CONNECT:
-        populateByReflection(level, command.getConnect(), traceDetails);
+        populateByReflection(command.getConnect(), traceDetails);
         break;
       case CONNECTED:
-        populateByReflection(level, command.getConnected(), traceDetails);
+        populateByReflection(command.getConnected(), traceDetails);
         break;
       case SUBSCRIBE:
-        populateByReflection(level, command.getSubscribe(), traceDetails);
+        populateByReflection(command.getSubscribe(), traceDetails);
         break;
       case PRODUCER:
-        populateByReflection(level, command.getProducer(), traceDetails);
+        populateByReflection(command.getProducer(), traceDetails);
         break;
       case SEND:
-        populateByReflection(level, command.getSend(), traceDetails);
+        populateByReflection(command.getSend(), traceDetails);
         break;
       case SEND_RECEIPT:
-        populateByReflection(level, command.getSendReceipt(), traceDetails);
+        populateByReflection(command.getSendReceipt(), traceDetails);
         break;
       case SEND_ERROR:
-        populateByReflection(level, command.getSendError(), traceDetails);
+        populateByReflection(command.getSendError(), traceDetails);
         break;
       case MESSAGE:
-        populateByReflection(level, command.getMessage(), traceDetails);
+        populateByReflection(command.getMessage(), traceDetails);
         break;
       case ACK:
-        populateByReflection(level, command.getAck(), traceDetails);
+        populateByReflection(command.getAck(), traceDetails);
         break;
       case FLOW:
-        populateByReflection(level, command.getFlow(), traceDetails);
+        populateByReflection(command.getFlow(), traceDetails);
         break;
       case UNSUBSCRIBE:
-        populateByReflection(level, command.getUnsubscribe(), traceDetails);
+        populateByReflection(command.getUnsubscribe(), traceDetails);
         break;
       case SUCCESS:
-        populateByReflection(level, command.getSuccess(), traceDetails);
+        populateByReflection(command.getSuccess(), traceDetails);
         break;
       case ERROR:
-        populateByReflection(level, command.getError(), traceDetails);
+        populateByReflection(command.getError(), traceDetails);
         break;
       case CLOSE_PRODUCER:
-        populateByReflection(level, command.getCloseProducer(), traceDetails);
+        populateByReflection(command.getCloseProducer(), traceDetails);
         break;
       case CLOSE_CONSUMER:
-        populateByReflection(level, command.getCloseConsumer(), traceDetails);
+        populateByReflection(command.getCloseConsumer(), traceDetails);
         break;
       case PRODUCER_SUCCESS:
-        populateByReflection(level, command.getProducerSuccess(), traceDetails);
+        populateByReflection(command.getProducerSuccess(), traceDetails);
         break;
       case PING:
-        populateByReflection(level, command.getPing(), traceDetails);
+        populateByReflection(command.getPing(), traceDetails);
         break;
       case PONG:
-        populateByReflection(level, command.getPong(), traceDetails);
+        populateByReflection(command.getPong(), traceDetails);
         break;
       case REDELIVER_UNACKNOWLEDGED_MESSAGES:
-        populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails);
+        populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
         break;
       case PARTITIONED_METADATA:
-        populateByReflection(level, command.getPartitionMetadata(), traceDetails);
+        populateByReflection(command.getPartitionMetadata(), traceDetails);
         break;
       case PARTITIONED_METADATA_RESPONSE:
-        populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails);
+        populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
         break;
       case LOOKUP:
-        populateByReflection(level, command.getLookupTopic(), traceDetails);
+        populateByReflection(command.getLookupTopic(), traceDetails);
         break;
       case LOOKUP_RESPONSE:
-        populateByReflection(level, command.getLookupTopicResponse(), traceDetails);
+        populateByReflection(command.getLookupTopicResponse(), traceDetails);
         break;
       case CONSUMER_STATS:
-        populateByReflection(level, command.getConsumerStats(), traceDetails);
+        populateByReflection(command.getConsumerStats(), traceDetails);
         break;
       case CONSUMER_STATS_RESPONSE:
-        populateByReflection(level, command.getConsumerStatsResponse(), traceDetails);
+        populateByReflection(command.getConsumerStatsResponse(), traceDetails);
         break;
       case REACHED_END_OF_TOPIC:
-        populateByReflection(level, command.getReachedEndOfTopic(), traceDetails);
+        populateByReflection(command.getReachedEndOfTopic(), traceDetails);
         break;
       case SEEK:
-        populateByReflection(level, command.getSeek(), traceDetails);
+        populateByReflection(command.getSeek(), traceDetails);
         break;
       case GET_LAST_MESSAGE_ID:
-        populateByReflection(level, command.getGetLastMessageId(), traceDetails);
+        populateByReflection(command.getGetLastMessageId(), traceDetails);
         break;
       case GET_LAST_MESSAGE_ID_RESPONSE:
-        populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails);
+        populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
         break;
       case ACTIVE_CONSUMER_CHANGE:
-        populateByReflection(level, command.getActiveConsumerChange(), traceDetails);
+        populateByReflection(command.getActiveConsumerChange(), traceDetails);
         break;
       case GET_TOPICS_OF_NAMESPACE:
-        populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails);
+        populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
         break;
       case GET_TOPICS_OF_NAMESPACE_RESPONSE:
-        populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails);
+        populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
         break;
       case GET_SCHEMA:
-        populateByReflection(level, command.getGetSchema(), traceDetails);
+        populateByReflection(command.getGetSchema(), traceDetails);
         break;
       case GET_SCHEMA_RESPONSE:
-        populateByReflection(level, command.getGetSchemaResponse(), traceDetails);
+        populateByReflection(command.getGetSchemaResponse(), traceDetails);
         break;
       case AUTH_CHALLENGE:
-        populateByReflection(level, command.getAuthChallenge(), traceDetails);
+        populateByReflection(command.getAuthChallenge(), traceDetails);
         break;
       case AUTH_RESPONSE:
-        populateByReflection(level, command.getAuthResponse(), traceDetails);
+        populateByReflection(command.getAuthResponse(), traceDetails);
         break;
       case ACK_RESPONSE:
-        populateByReflection(level, command.getAckResponse(), traceDetails);
+        populateByReflection(command.getAckResponse(), traceDetails);
         break;
       case GET_OR_CREATE_SCHEMA:
-        populateByReflection(level, command.getGetOrCreateSchema(), traceDetails);
+        populateByReflection(command.getGetOrCreateSchema(), traceDetails);
         break;
       case GET_OR_CREATE_SCHEMA_RESPONSE:
-        populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails);
+        populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
         break;
       case NEW_TXN:
-        populateByReflection(level, command.getNewTxn(), traceDetails);
+        populateByReflection(command.getNewTxn(), traceDetails);
         break;
       case NEW_TXN_RESPONSE:
-        populateByReflection(level, command.getNewTxnResponse(), traceDetails);
+        populateByReflection(command.getNewTxnResponse(), traceDetails);
         break;
       case ADD_PARTITION_TO_TXN:
-        populateByReflection(level, command.getAddPartitionToTxn(), traceDetails);
+        populateByReflection(command.getAddPartitionToTxn(), traceDetails);
         break;
       case ADD_PARTITION_TO_TXN_RESPONSE:
-        populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails);
+        populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
         break;
       case ADD_SUBSCRIPTION_TO_TXN:
-        populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails);
+        populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
         break;
       case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
-        populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails);
+        populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
         break;
       case END_TXN:
-        populateByReflection(level, command.getEndTxn(), traceDetails);
+        populateByReflection(command.getEndTxn(), traceDetails);
         break;
       case END_TXN_RESPONSE:
-        populateByReflection(level, command.getEndTxnResponse(), traceDetails);
+        populateByReflection(command.getEndTxnResponse(), traceDetails);
         break;
       case END_TXN_ON_PARTITION:
-        populateByReflection(level, command.getEndTxnOnPartition(), traceDetails);
+        populateByReflection(command.getEndTxnOnPartition(), traceDetails);
         break;
       case END_TXN_ON_PARTITION_RESPONSE:
-        populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails);
+        populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
         break;
       case END_TXN_ON_SUBSCRIPTION:
-        populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails);
+        populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
         break;
       case END_TXN_ON_SUBSCRIPTION_RESPONSE:
-        populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails);
+        populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
         break;
       case TC_CLIENT_CONNECT_REQUEST:
-        populateByReflection(level, command.getTcClientConnectRequest(), traceDetails);
+        populateByReflection(command.getTcClientConnectRequest(), traceDetails);
         break;
       case TC_CLIENT_CONNECT_RESPONSE:
-        populateByReflection(level, command.getTcClientConnectResponse(), traceDetails);
+        populateByReflection(command.getTcClientConnectResponse(), traceDetails);
         break;
       case WATCH_TOPIC_LIST:
-        populateByReflection(level, command.getWatchTopicList(), traceDetails);
+        populateByReflection(command.getWatchTopicList(), traceDetails);
         break;
       case WATCH_TOPIC_LIST_SUCCESS:
-        populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails);
+        populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
         break;
       case WATCH_TOPIC_UPDATE:
-        populateByReflection(level, command.getWatchTopicUpdate(), traceDetails);
+        populateByReflection(command.getWatchTopicUpdate(), traceDetails);
         break;
       case WATCH_TOPIC_LIST_CLOSE:
-        populateByReflection(level, command.getWatchTopicListClose(), traceDetails);
+        populateByReflection(command.getWatchTopicListClose(), traceDetails);
         break;
       case TOPIC_MIGRATED:
-        populateByReflection(level, command.getTopicMigrated(), traceDetails);
+        populateByReflection(command.getTopicMigrated(), traceDetails);
         break;
       default:
         log.error("Unknown command type: {}", command.getType());
@@ -292,7 +312,7 @@ private static void populateCommandDetails(
     }
   }
 
-  private static final Set<String> fullTraceFields =
+  private static final Set<String> skipTraceFields =
       Sets.newHashSet(
           "authdata",
           "authmethod",
@@ -302,8 +322,7 @@ private static void populateCommandDetails(
           "originalprincipal",
           "schema");
 
-  private static void populateByReflection(
-      TraceLevel level, Object command, Map<String, Object> traceDetails) {
+  private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
     if (command == null) {
       return;
     }
@@ -320,7 +339,7 @@ private static void populateByReflection(
                 return false;
               }
               String fieldName = method.getName().substring(3);
-              return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase());
+              return !skipTraceFields.contains(fieldName.toLowerCase());
             })
         .filter(
             method -> {
@@ -356,12 +375,12 @@ private static void populateByReflection(
                 if (value instanceof byte[]
                     || value instanceof ByteBuf
                     || value instanceof ByteBuffer) {
-                  int size = 0;
+                  final int size;
                   if (value instanceof byte[]) {
                     size = ((byte[]) value).length;
                   } else if (value instanceof ByteBuf) {
                     size = ((ByteBuf) value).readableBytes();
-                  } else if (value instanceof ByteBuffer) {
+                  } else {
                     size = ((ByteBuffer) value).remaining();
                   }
                   traceDetails.put(fieldName + "_size", size);
@@ -373,7 +392,7 @@ private static void populateByReflection(
                     .getCanonicalName()
                     .contains("org.apache.pulsar.common.api.proto")) {
                   Map<String, Object> details = new TreeMap<>();
-                  populateByReflection(level, value, details);
+                  populateByReflection(value, details);
                   traceDetails.put(fieldName, details);
                 } else {
                   traceDetails.put(fieldName, value);
@@ -384,53 +403,35 @@ private static void populateByReflection(
             });
   }
 
-  public static Map<String, Object> getConnectionDetails(TraceLevel level, ServerCnx cnx) {
+  public static Map<String, Object> getConnectionDetails(ServerCnx cnx) {
     if (cnx == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateConnectionDetails(level, cnx, details);
+    populateConnectionDetails(cnx, details);
     return details;
   }
 
-  private static void populateConnectionDetails(
-      TraceLevel level, ServerCnx cnx, Map<String, Object> traceDetails) {
+  private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object> traceDetails) {
     if (cnx == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("clientAddress", cnx.clientAddress().toString());
-        traceDetails.put("authRole", cnx.getAuthRole());
-        break;
-      case BASIC:
-        populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails);
-
-        traceDetails.put("clientVersion", cnx.getClientVersion());
-        traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
-        break;
-      case FULL:
-        populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails);
-
-        traceDetails.put("authMethod", cnx.getAuthMethod());
-        traceDetails.put(
-            "authMethodName",
-            cnx.getAuthenticationProvider() == null
-                ? "no provider"
-                : cnx.getAuthenticationProvider().getAuthMethodName());
-
-        AuthenticationDataSource authData = cnx.getAuthenticationData();
-        if (authData != null) {
-          traceDetails.put("authData", getAuthDataDetails(authData));
-        }
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
+    traceDetails.put("clientAddress", cnx.clientAddress());
+    traceDetails.put("authRole", cnx.getAuthRole());
+    traceDetails.put("clientVersion", cnx.getClientVersion());
+    traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
+    traceDetails.put("authMethod", cnx.getAuthMethod());
+    traceDetails.put(
+        "authMethodName",
+        cnx.getAuthenticationProvider() == null
+            ? "no provider"
+            : cnx.getAuthenticationProvider().getAuthMethodName());
+
+    AuthenticationDataSource authData = cnx.getAuthenticationData();
+    if (authData != null) {
+      traceDetails.put("authData", getAuthDataDetails(authData));
     }
   }
 
@@ -463,260 +464,178 @@ private static void populateAuthDataDetails(
     }
   }
 
-  public static Map<String, Object> getSubscriptionDetails(TraceLevel level, Subscription sub) {
+  public static Map<String, Object> getSubscriptionDetails(Subscription sub) {
     if (sub == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateSubscriptionDetails(level, sub, details);
+    populateSubscriptionDetails(sub, details);
     return details;
   }
 
   private static void populateSubscriptionDetails(
-      TraceLevel level, Subscription sub, Map<String, Object> traceDetails) {
+      Subscription sub, Map<String, Object> traceDetails) {
     if (sub == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("name", sub.getName());
-        traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-        traceDetails.put("type", sub.getType().name());
-        break;
-      case BASIC:
-        populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails);
-
-        if (sub.getConsumers() != null) {
-          traceDetails.put("numberOfConsumers", sub.getConsumers().size());
-          traceDetails.put(
-              "namesOfConsumers",
-              sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
-        }
+    traceDetails.put("name", sub.getName());
+    traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+    traceDetails.put("type", sub.getType().name());
 
-        break;
-      case FULL:
-        populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails);
-
-        traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
+    if (sub.getConsumers() != null) {
+      traceDetails.put("numberOfConsumers", sub.getConsumers().size());
+      traceDetails.put(
+          "namesOfConsumers",
+          sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
     }
+
+    traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
   }
 
-  public static Map<String, Object> getConsumerDetails(TraceLevel level, Consumer consumer) {
+  public static Map<String, Object> getConsumerDetails(Consumer consumer) {
     if (consumer == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateConsumerDetails(level, consumer, details);
+    populateConsumerDetails(consumer, details);
     return details;
   }
 
-  private static void populateConsumerDetails(
-      TraceLevel level, Consumer consumer, Map<String, Object> traceDetails) {
+  private static void populateConsumerDetails(Consumer consumer, Map<String, Object> traceDetails) {
     if (consumer == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("name", consumer.consumerName());
-        traceDetails.put("consumerId", consumer.consumerId());
-        Subscription sub = consumer.getSubscription();
-        if (sub != null) {
-          traceDetails.put("subscriptionName", sub.getName());
-          traceDetails.put(
-              "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-        }
-        break;
-      case BASIC:
-        populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails);
+    traceDetails.put("name", consumer.consumerName());
+    traceDetails.put("consumerId", consumer.consumerId());
+    Subscription sub = consumer.getSubscription();
+    if (sub != null) {
+      traceDetails.put("subscriptionName", sub.getName());
+      traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+    }
 
-        traceDetails.put("priorityLevel", consumer.getPriorityLevel());
-        traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
-        traceDetails.put("clientAddress", consumer.getClientAddress());
-        break;
-      case FULL:
-        populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails);
+    traceDetails.put("priorityLevel", consumer.getPriorityLevel());
+    traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
+    traceDetails.put("clientAddress", consumer.getClientAddress());
 
-        traceDetails.put("metadata", consumer.getMetadata());
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
+    traceDetails.put("metadata", consumer.getMetadata());
   }
 
-  public static Map<String, Object> getProducerDetails(
-      TraceLevel level, Producer producer, boolean traceSchema) {
+  public static Map<String, Object> getProducerDetails(Producer producer, boolean traceSchema) {
     if (producer == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateProducerDetails(level, producer, details, traceSchema);
+    populateProducerDetails(producer, details, traceSchema);
     return details;
   }
 
   private static void populateProducerDetails(
-      TraceLevel level, Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
+      Producer producer, Map<String, Object> traceDetails, boolean traceSchema) {
     if (producer == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("producerId", producer.getProducerId());
-        traceDetails.put("producerName", producer.getProducerName());
-        traceDetails.put(
-            "accessMode",
-            producer.getAccessMode() == null ? null : producer.getAccessMode().name());
-        if (producer.getTopic() != null) {
-          traceDetails.put(
-              "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
-        }
-        break;
-      case BASIC:
-        populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema);
+    traceDetails.put("producerId", producer.getProducerId());
+    traceDetails.put("producerName", producer.getProducerName());
+    traceDetails.put(
+        "accessMode", producer.getAccessMode() == null ? null : producer.getAccessMode().name());
+    if (producer.getTopic() != null) {
+      traceDetails.put(
+          "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+    }
 
-        traceDetails.put("clientAddress", producer.getClientAddress());
-        break;
-      case FULL:
-        populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema);
+    traceDetails.put("clientAddress", producer.getClientAddress());
 
-        traceDetails.put("metadata", producer.getMetadata());
+    traceDetails.put("metadata", producer.getMetadata());
 
-        if (traceSchema && producer.getSchemaVersion() != null) {
-          final String schemaVersion;
-          if (producer.getSchemaVersion() == SchemaVersion.Empty) {
-            schemaVersion = "Empty";
-          } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
-            schemaVersion = "Latest";
-          } else {
-            schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
-          }
-          traceDetails.put("schemaVersion", schemaVersion);
-        }
-        traceDetails.put("remoteCluster", producer.getRemoteCluster());
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
+    if (traceSchema && producer.getSchemaVersion() != null) {
+      final String schemaVersion;
+      if (producer.getSchemaVersion() == SchemaVersion.Empty) {
+        schemaVersion = "Empty";
+      } else if (producer.getSchemaVersion() == SchemaVersion.Latest) {
+        schemaVersion = "Latest";
+      } else {
+        schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes());
+      }
+      traceDetails.put("schemaVersion", schemaVersion);
     }
+    traceDetails.put("remoteCluster", producer.getRemoteCluster());
   }
 
-  public static Map<String, Object> getMessageMetadataDetails(
-      TraceLevel level, MessageMetadata msgMetadata) {
+  public static Map<String, Object> getMessageMetadataDetails(MessageMetadata msgMetadata) {
     if (msgMetadata == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateMessageMetadataDetails(level, msgMetadata, details);
+    populateMessageMetadataDetails(msgMetadata, details);
     return details;
   }
 
   private static void populateMessageMetadataDetails(
-      TraceLevel level, MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
+      MessageMetadata msgMetadata, Map<String, Object> traceDetails) {
     if (msgMetadata == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        if (msgMetadata.hasPartitionKey()) {
-          traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
-        }
-        if (msgMetadata.hasSequenceId()) {
-          traceDetails.put("sequenceId", msgMetadata.getSequenceId());
-        }
-        if (msgMetadata.hasProducerName()) {
-          traceDetails.put("producerName", msgMetadata.getProducerName());
-        }
-        break;
-      case BASIC:
-        populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails);
-
-        if (msgMetadata.hasUncompressedSize()) {
-          traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
-        }
-        if (msgMetadata.hasNumMessagesInBatch()) {
-          traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
-        }
-        traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
-        break;
-      case FULL:
-        populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails);
-
-        if (msgMetadata.hasPublishTime()) {
-          traceDetails.put("publishTime", msgMetadata.getPublishTime());
-        }
-        if (msgMetadata.hasEventTime()) {
-          traceDetails.put("eventTime", msgMetadata.getEventTime());
-        }
-        if (msgMetadata.hasReplicatedFrom()) {
-          traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
-        }
-        if (msgMetadata.hasUuid()) {
-          traceDetails.put("uuid", msgMetadata.getUuid());
-        }
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
+    if (msgMetadata.hasPartitionKey()) {
+      traceDetails.put("partitionKey", msgMetadata.getPartitionKey());
+    }
+    if (msgMetadata.hasSequenceId()) {
+      traceDetails.put("sequenceId", msgMetadata.getSequenceId());
+    }
+    if (msgMetadata.hasProducerName()) {
+      traceDetails.put("producerName", msgMetadata.getProducerName());
+    }
+
+    if (msgMetadata.hasUncompressedSize()) {
+      traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
+    }
+    if (msgMetadata.hasNumMessagesInBatch()) {
+      traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch());
+    }
+    traceDetails.put("serializedSize", msgMetadata.getSerializedSize());
+
+    if (msgMetadata.hasPublishTime()) {
+      traceDetails.put("publishTime", msgMetadata.getPublishTime());
+    }
+    if (msgMetadata.hasEventTime()) {
+      traceDetails.put("eventTime", msgMetadata.getEventTime());
+    }
+    if (msgMetadata.hasReplicatedFrom()) {
+      traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom());
+    }
+    if (msgMetadata.hasUuid()) {
+      traceDetails.put("uuid", msgMetadata.getUuid());
     }
   }
 
-  public static Map<String, Object> getEntryDetails(
-      TraceLevel level, Entry entry, int maxBinaryDataLength) {
+  public static Map<String, Object> getEntryDetails(Entry entry, int maxBinaryDataLength) {
     if (entry == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateEntryDetails(level, entry, details, maxBinaryDataLength);
+    populateEntryDetails(entry, details, maxBinaryDataLength);
     return details;
   }
 
   private static void populateEntryDetails(
-      TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+      Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
     if (entry == null) {
       return;
     }
 
-    switch (level) {
-      case MINIMAL:
-        traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
-        break;
-      case BASIC:
-        populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength);
+    traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
 
-        traceDetails.put("length", entry.getLength());
-        break;
-      case FULL:
-        populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength);
+    traceDetails.put("length", entry.getLength());
 
-        traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
-        break;
-      case NONE:
-        break;
-      default:
-        log.warn("Unknown tracing level: {}", level);
-        break;
-    }
+    traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
   }
 
   public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 220fee87..fbae35b2 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -32,11 +32,15 @@ class TracingUtilsTest {
   private Tracer mockTracer =
       new Tracer() {
         @Override
-        public void trace(String msg) {
+        public void trace(EventReasons reason, String msg) {
           traces.add(msg);
         }
       };
 
+  private static void trace(Tracer mockTracer, String msg, Map<String, Object> traceDetails) {
+    TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails);
+  }
+
   @Test
   void traceTest() {
     traces.clear();

From 8e8c866c6615c4b5a39ef92d594b630e74f1af26 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 16:51:21 -0700
Subject: [PATCH 21/29] resolve hostname of the client

---
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 50 +++++++++++++++++--
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index d995387a..5caa7e36 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -18,9 +18,14 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Sets;
 import io.netty.buffer.ByteBuf;
 import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -88,6 +93,44 @@ public enum TraceLevel {
     ON
   }
 
+  private static final LoadingCache<String, String> ipResolverCache =
+      CacheBuilder.newBuilder()
+          .maximumSize(10_000L)
+          .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+          .build(
+              new CacheLoader<String, String>() {
+                public String load(String clientAddress) {
+                  // Dn resolution can be slow in some cases
+                  // and we do not want to create too many requests to DNS,
+                  // so we cache the result
+                  log.info("resolving DNS for {}", clientAddress);
+                  try {
+                    InetAddress address = InetAddress.getByName(clientAddress);
+                    String hostName = address.getCanonicalHostName();
+                    if (log.isDebugEnabled()) {
+                      log.debug("Resolved DNS for {} to {}", clientAddress, hostName);
+                    }
+                    return hostName;
+                  } catch (UnknownHostException e) {
+                    log.error("Failed to resolve DNS for {}", clientAddress, e);
+                    return clientAddress;
+                  }
+                }
+              });
+
+  public static String hostNameOf(String clientAddress) {
+    if (clientAddress == null || clientAddress.isEmpty()) {
+      return "unknown/null";
+    }
+
+    try {
+      return ipResolverCache.get(clientAddress);
+    } catch (Throwable t) {
+      log.error("Failed to resolve DNS for {}", clientAddress, t);
+      return clientAddress;
+    }
+  }
+
   public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
     trace(SLF4J_TRACER, reason, message, traceDetails);
   }
@@ -418,7 +461,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
       return;
     }
 
-    traceDetails.put("clientAddress", cnx.clientAddress());
+    traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress()));
+    traceDetails.put("clientSocket", cnx.clientAddress());
     traceDetails.put("authRole", cnx.getAuthRole());
     traceDetails.put("clientVersion", cnx.getClientVersion());
     traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
@@ -519,7 +563,7 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
 
     traceDetails.put("priorityLevel", consumer.getPriorityLevel());
     traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
-    traceDetails.put("clientAddress", consumer.getClientAddress());
+    traceDetails.put("clientAddress", hostNameOf(consumer.getClientAddress()));
 
     traceDetails.put("metadata", consumer.getMetadata());
   }
@@ -549,7 +593,7 @@ private static void populateProducerDetails(
           "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
     }
 
-    traceDetails.put("clientAddress", producer.getClientAddress());
+    traceDetails.put("clientAddress", hostNameOf(producer.getClientAddress()));
 
     traceDetails.put("metadata", producer.getMetadata());
 

From f16ac60b5e0d2c012d5f7129884126dea7c18c57 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 17:55:09 -0700
Subject: [PATCH 22/29] parse payload out of the ByteBuf

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java |  6 ++-
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 47 +++++++++++++++++--
 2 files changed, 47 insertions(+), 6 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 68e34beb..bdcf22ac 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -466,7 +466,11 @@ public void onMessagePublish(
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
-    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
+
+    Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+    traceByteBuf(
+        "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+    traceDetails.put("headersAndPayload", headersAndPayloadDetails);
 
     trace(EventReasons.MESSAGE, "Message publish", traceDetails);
   }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 5caa7e36..12af1721 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -27,6 +27,7 @@
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -45,7 +46,10 @@
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.compression.CompressionCodec;
+import org.apache.pulsar.common.compression.CompressionCodecProvider;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.schema.SchemaVersion;
 
 @Slf4j
@@ -709,12 +713,45 @@ private static void populatePublishContext(
   public static void traceByteBuf(
       String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
     if (buf == null || maxBinaryDataLength <= 0) return;
+    try {
 
-    if (buf.readableBytes() < maxBinaryDataLength) {
-      traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
-    } else {
-      traceDetails.put(
-          key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()));
+      final ByteBuf metadataAndPayload = buf.retainedDuplicate();
+      ByteBuf uncompressedPayload = null;
+      try {
+        // advance readerIndex
+        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
+
+        // todo: do we need to trace this metadata?
+        populateMessageMetadataDetails(metadata, traceDetails);
+
+        // Decode if needed
+        CompressionCodec codec =
+            CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
+        uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
+
+        // todo: does this require additional steps if messages are batched?
+        if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) {
+          String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8);
+          traceDetails.put(key, dataAsString);
+        } else {
+          String dataAsString =
+              uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8);
+          traceDetails.put(key, dataAsString + "...");
+        }
+      } finally {
+        metadataAndPayload.release();
+        if (uncompressedPayload != null) {
+          uncompressedPayload.release();
+        }
+      }
+    } catch (Throwable t) {
+      log.error("Failed to convert ByteBuf to string", t);
+      if (buf.readableBytes() < maxBinaryDataLength + 3) {
+        traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
+      } else {
+        traceDetails.put(
+            key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "...");
+      }
     }
   }
 }

From 277e0b45f20b9c4503e14b7efb6680d1c603568e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Thu, 2 May 2024 18:02:03 -0700
Subject: [PATCH 23/29] fix test, some wording

---
 .../datastax/oss/pulsar/jms/tracing/BrokerTracing.java    | 8 ++++++--
 .../com/datastax/oss/pulsar/jms/tracing/TracingUtils.java | 2 +-
 .../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java | 4 ++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index bdcf22ac..dfbfdec7 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -470,7 +470,7 @@ public void onMessagePublish(
     Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
     traceByteBuf(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
-    traceDetails.put("headersAndPayload", headersAndPayloadDetails);
+    traceDetails.put("payload", headersAndPayloadDetails);
 
     trace(EventReasons.MESSAGE, "Message publish", traceDetails);
   }
@@ -508,7 +508,11 @@ public void messageDispatched(
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
-    traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength);
+
+    Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+    traceByteBuf(
+            "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+    traceDetails.put("payload", headersAndPayloadDetails);
 
     trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
   }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 12af1721..64e4aa92 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -683,7 +683,7 @@ private static void populateEntryDetails(
 
     traceDetails.put("length", entry.getLength());
 
-    traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
+    traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
   }
 
   public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index fbae35b2..71e01bba 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -118,7 +118,7 @@ void traceByteBufTest() {
     traceDetails.clear();
     traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
     assertEquals(1, traceDetails.size());
-    assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length());
-    assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102"));
+    assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
+    assertTrue(((String) traceDetails.get("key")).startsWith("0x000102"));
   }
 }

From 87c640897004ed0538d07e0e00a888258fe65762 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 3 May 2024 15:02:07 -0700
Subject: [PATCH 24/29] WIP reduce verbosity

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 47 ++++++++++++++-----
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 30 ++++++------
 2 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dfbfdec7..0af2d244 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -64,6 +64,9 @@
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.intercept.InterceptException;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
+import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
+import org.apache.pulsar.common.util.DateFormatter;
 import org.jetbrains.annotations.NotNull;
 
 @Slf4j
@@ -351,7 +354,6 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
     if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
@@ -365,10 +367,17 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
 
+    PublisherStatsImpl stats = producer.getStats();
+    traceDetails.put("connectedSince", stats.getConnectedSince());
+    traceDetails.put("closedAt", DateFormatter.now());
+    traceDetails.put("averageMsgSize", stats.getAverageMsgSize());
+    traceDetails.put("msgRateIn", stats.getMsgRateIn());
+    traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
+    // no message count in stats? stats.getCount() is not it
+
     trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
   }
 
@@ -379,7 +388,6 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
     if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
@@ -394,11 +402,22 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     if (traceLevel == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
 
+    ConsumerStatsImpl stats = consumer.getStats();
+
+    traceDetails.put("connectedSince", stats.getConnectedSince());
+    traceDetails.put("closedAt", DateFormatter.now());
+    traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
+    traceDetails.put("msgRateOut", stats.getMsgRateOut());
+    traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut());
+    traceDetails.put("msgOutCounter", stats.getMsgOutCounter());
+    traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
+    traceDetails.put("unackedMessages", stats.getUnackedMessages());
+    traceDetails.put("messageAckRate", stats.getMessageAckRate());
+
     trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
   }
 
@@ -452,7 +471,7 @@ public void beforeSendMessage(
     traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
     traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
 
-    trace(EventReasons.MESSAGE, "Before sending message", traceDetails);
+    trace(EventReasons.MESSAGE, "Message read", traceDetails);
   }
 
   public void onMessagePublish(
@@ -472,7 +491,7 @@ public void onMessagePublish(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "Message publish", traceDetails);
+    trace(EventReasons.MESSAGE, "Message received", traceDetails);
   }
 
   public void messageProduced(
@@ -493,7 +512,7 @@ public void messageProduced(
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
-    trace(EventReasons.MESSAGE, "Message produced", traceDetails);
+    trace(EventReasons.MESSAGE, "Message stored", traceDetails);
   }
 
   public void messageDispatched(
@@ -504,17 +523,18 @@ public void messageDispatched(
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("consumer", getConsumerDetails(consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+    if (consumer != null) {
+      traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+    }
     traceDetails.put("messageId", ledgerId + ":" + entryId);
 
     Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
     traceByteBuf(
-            "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
+        "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "After dispatching message", traceDetails);
+    trace(EventReasons.MESSAGE, "Message dispatched", traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
@@ -524,9 +544,10 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("consumer", getConsumerDetails(consumer));
-    traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+    if (consumer != null) {
+      traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+    }
 
     Map<String, Object> ackDetails = new TreeMap<>();
     if (ackCmd.hasAckType()) {
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 64e4aa92..25f933e1 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -34,6 +34,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.Entry;
@@ -100,7 +101,7 @@ public enum TraceLevel {
   private static final LoadingCache<String, String> ipResolverCache =
       CacheBuilder.newBuilder()
           .maximumSize(10_000L)
-          .concurrencyLevel(Runtime.getRuntime().availableProcessors())
+          .expireAfterWrite(4, TimeUnit.HOURS)
           .build(
               new CacheLoader<String, String>() {
                 public String load(String clientAddress) {
@@ -464,18 +465,15 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
     if (cnx == null) {
       return;
     }
-
-    traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress()));
-    traceDetails.put("clientSocket", cnx.clientAddress());
+    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
     traceDetails.put("authRole", cnx.getAuthRole());
+    traceDetails.put("principal", cnx.getPrincipal());
     traceDetails.put("clientVersion", cnx.getClientVersion());
     traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
     traceDetails.put("authMethod", cnx.getAuthMethod());
-    traceDetails.put(
-        "authMethodName",
-        cnx.getAuthenticationProvider() == null
-            ? "no provider"
-            : cnx.getAuthenticationProvider().getAuthMethodName());
+    if (cnx.getAuthenticationProvider() != null) {
+      traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
+    }
 
     AuthenticationDataSource authData = cnx.getAuthenticationData();
     if (authData != null) {
@@ -534,10 +532,10 @@ private static void populateSubscriptionDetails(
 
     if (sub.getConsumers() != null) {
       traceDetails.put("numberOfConsumers", sub.getConsumers().size());
-      traceDetails.put(
-          "namesOfConsumers",
-          sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList()));
     }
+    traceDetails.put("isReplicated", sub.isReplicated());
+    traceDetails.put("numberOfEntriesDelayed", sub.getNumberOfEntriesDelayed());
+    traceDetails.put("numberOfEntriesInBacklog", sub.getNumberOfEntriesInBacklog(false));
 
     traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties());
   }
@@ -567,9 +565,11 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
 
     traceDetails.put("priorityLevel", consumer.getPriorityLevel());
     traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
-    traceDetails.put("clientAddress", hostNameOf(consumer.getClientAddress()));
+    traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress()));
 
     traceDetails.put("metadata", consumer.getMetadata());
+    traceDetails.put("unackedMessages", consumer.getUnackedMessages());
+    traceDetails.put("authRole", consumer.cnx().getAuthRole());
   }
 
   public static Map<String, Object> getProducerDetails(Producer producer, boolean traceSchema) {
@@ -597,7 +597,7 @@ private static void populateProducerDetails(
           "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
     }
 
-    traceDetails.put("clientAddress", hostNameOf(producer.getClientAddress()));
+    traceDetails.put("clientHost", hostNameOf(producer.getClientAddress()));
 
     traceDetails.put("metadata", producer.getMetadata());
 
@@ -613,6 +613,8 @@ private static void populateProducerDetails(
       traceDetails.put("schemaVersion", schemaVersion);
     }
     traceDetails.put("remoteCluster", producer.getRemoteCluster());
+
+    traceDetails.put("authRole", producer.getCnx().getAuthRole());
   }
 
   public static Map<String, Object> getMessageMetadataDetails(MessageMetadata msgMetadata) {

From 398fa3f490e4988cef4cf28d7121667bec0a7f21 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Mon, 6 May 2024 16:28:49 -0700
Subject: [PATCH 25/29] reduced traces verbosity, added some details

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 108 +++++++++++++-----
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  14 +--
 2 files changed, 88 insertions(+), 34 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 0af2d244..5023cec9 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -21,7 +21,6 @@
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
@@ -333,6 +332,47 @@ private TraceLevel getTracingLevel(Producer producer) {
     }
   }
 
+  private static void addMinimumProducerDetails(
+      Producer producer, Map<String, Object> traceDetails) {
+    if (producer == null) return;
+
+    traceDetails.put("producerId", producer.getProducerId());
+    traceDetails.put("producerName", producer.getProducerName());
+    if (producer.getAccessMode() != null) {
+      traceDetails.put("accessMode", producer.getAccessMode().name());
+    }
+    traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress()));
+    if (producer.getTopic() != null) {
+      traceDetails.put(
+          "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
+    }
+    traceDetails.put("authRole", producer.getCnx().getAuthRole());
+  }
+
+  private static void addMinimumConsumerSubscriptionDetails(
+      Consumer consumer, Map<String, Object> traceDetails) {
+    if (consumer == null) return;
+
+    addMinimumConsumerSubscriptionDetails(consumer, consumer.getSubscription(), traceDetails);
+  }
+
+  private static void addMinimumConsumerSubscriptionDetails(
+      Consumer consumer, Subscription subscription, Map<String, Object> traceDetails) {
+    if (consumer != null) {
+      traceDetails.put("consumerName", consumer.consumerName());
+      traceDetails.put("consumerId", consumer.consumerId());
+      traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress()));
+      traceDetails.put("authRole", consumer.cnx().getAuthRole());
+    }
+
+    if (subscription != null) {
+      traceDetails.put("subscriptionName", subscription.getName());
+      traceDetails.put(
+          "topicName", TopicName.get(subscription.getTopicName()).getPartitionedTopicName());
+      traceDetails.put("subscriptionType", subscription.getType().name());
+    }
+  }
+
   /* ***************************
    **  Administrative events
    ******************************/
@@ -344,6 +384,8 @@ public void onConnectionCreated(ServerCnx cnx) {
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
+
     trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
   }
 
@@ -356,6 +398,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
   }
@@ -369,6 +412,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     PublisherStatsImpl stats = producer.getStats();
     traceDetails.put("connectedSince", stats.getConnectedSince());
@@ -391,6 +435,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
   }
@@ -405,9 +450,9 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     ConsumerStatsImpl stats = consumer.getStats();
-
     traceDetails.put("connectedSince", stats.getConnectedSince());
     traceDetails.put("closedAt", DateFormatter.now());
     traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
@@ -417,6 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
     traceDetails.put("unackedMessages", stats.getUnackedMessages());
     traceDetails.put("messageAckRate", stats.getMessageAckRate());
+    traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
 
     trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
   }
@@ -446,6 +492,7 @@ public void onConnectionClosed(ServerCnx cnx) {
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
   }
@@ -466,12 +513,12 @@ public void beforeSendMessage(
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("subscription", getSubscriptionDetails(subscription));
-    traceDetails.put("consumer", getConsumerDetails(consumer));
+
+    addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
+
     traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
-    traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata));
 
-    trace(EventReasons.MESSAGE, "Message read", traceDetails);
+    trace(EventReasons.MESSAGE, "read", traceDetails);
   }
 
   public void onMessagePublish(
@@ -483,7 +530,9 @@ public void onMessagePublish(
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
+
+    addMinimumProducerDetails(producer, traceDetails);
+
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
 
     Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
@@ -491,7 +540,7 @@ public void onMessagePublish(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "Message received", traceDetails);
+    trace(EventReasons.MESSAGE, "received", traceDetails);
   }
 
   public void messageProduced(
@@ -507,12 +556,12 @@ public void messageProduced(
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
-    traceDetails.put("producer", getProducerDetails(producer, traceSchema));
+    addMinimumProducerDetails(producer, traceDetails);
+
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
-    trace(EventReasons.MESSAGE, "Message stored", traceDetails);
+    trace(EventReasons.MESSAGE, "stored", traceDetails);
   }
 
   public void messageDispatched(
@@ -523,10 +572,7 @@ public void messageDispatched(
     if (level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("consumer", getConsumerDetails(consumer));
-    if (consumer != null) {
-      traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
-    }
+    addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
     traceDetails.put("messageId", ledgerId + ":" + entryId);
 
     Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
@@ -534,31 +580,41 @@ public void messageDispatched(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "Message dispatched", traceDetails);
+    trace(EventReasons.MESSAGE, "dispatched", traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
 
     TraceLevel level = getTracingLevel(consumer);
-    if (level == TraceLevel.OFF) return;
+    if (consumer != null && level == TraceLevel.OFF) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("consumer", getConsumerDetails(consumer));
-    if (consumer != null) {
-      traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
+
+    addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
+
+    if (consumer == null) {
+      // ack with empty consumer == message filtered by JMSFilter
+      traceDetails.put("reason", "filtered by JMSFilter");
+    } else {
+      // todo: am I right that unacked/nacked messages never go through broker interceptor?
+      // in this case we need consumer interceptor to track nacks
+      traceDetails.put("reason", "acked");
+    }
+
+    if (consumer != null && consumer.getSubscription() != null) {
+      Subscription sub = consumer.getSubscription();
+      traceDetails.put("subscriptionName", sub.getName());
+      traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
+      traceDetails.put("subscriptionType", sub.getType().name());
     }
 
     Map<String, Object> ackDetails = new TreeMap<>();
     if (ackCmd.hasAckType()) {
       ackDetails.put("type", ackCmd.getAckType().name());
-    } else {
-      ackDetails.put("type", "NOT SET");
     }
     if (ackCmd.hasConsumerId()) {
-      ackDetails.put("consumerId", ackCmd.getConsumerId());
-    } else {
-      ackDetails.put("consumerId", "NOT SET");
+      ackDetails.put("ackConsumerId", ackCmd.getConsumerId());
     }
     ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
     ackDetails.put(
@@ -579,7 +635,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
 
     traceDetails.put("ack", ackDetails);
 
-    trace(EventReasons.MESSAGE, "Message acked", traceDetails);
+    trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
   }
 
   @NotNull
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 25f933e1..f0317a0c 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -475,6 +475,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
       traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName());
     }
 
+    traceDetails.put("state", cnx.getState());
+
     AuthenticationDataSource authData = cnx.getAuthenticationData();
     if (authData != null) {
       traceDetails.put("authData", getAuthDataDetails(authData));
@@ -639,10 +641,6 @@ private static void populateMessageMetadataDetails(
     if (msgMetadata.hasSequenceId()) {
       traceDetails.put("sequenceId", msgMetadata.getSequenceId());
     }
-    if (msgMetadata.hasProducerName()) {
-      traceDetails.put("producerName", msgMetadata.getProducerName());
-    }
-
     if (msgMetadata.hasUncompressedSize()) {
       traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize());
     }
@@ -703,12 +701,12 @@ private static void populatePublishContext(
     traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
     traceDetails.put("isChunked", publishContext.isChunked());
     traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
-
     traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
     traceDetails.put("msgSize", publishContext.getMsgSize());
-    traceDetails.put("producerName", publishContext.getProducerName());
-    traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
-    traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+    if (publishContext.getOriginalProducerName() != null) {
+      traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+      traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+    }
     traceDetails.put("sequenceId", publishContext.getSequenceId());
   }
 

From c74a95bbe3ac5f8d7f029d499581dbbbc1db850b Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Tue, 7 May 2024 18:31:12 -0700
Subject: [PATCH 26/29] tweaks, servlet tracing

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++++-----
 .../oss/pulsar/jms/tracing/TracingUtils.java  |  76 +++++++++++
 2 files changed, 168 insertions(+), 29 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index 5023cec9..cc34cf57 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -24,6 +24,7 @@
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
 
@@ -33,7 +34,6 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import io.netty.buffer.ByteBuf;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
@@ -44,11 +44,13 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import javax.servlet.ServletException;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.Entry;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.intercept.BrokerInterceptor;
 import org.apache.pulsar.broker.service.Consumer;
@@ -83,9 +85,9 @@ public class BrokerTracing implements BrokerInterceptor {
   private static void loadEnabledEvents(
       PulsarService pulsarService, Set<EventReasons> enabledEvents) {
     Properties props = pulsarService.getConfiguration().getProperties();
-    if (props.contains("jmsTracingEventList")) {
+    if (props.containsKey("jmsTracingEventList")) {
       String events = props.getProperty("jmsTracingEventList", "");
-      log.debug("read jmsTracingEventList: {}", events);
+      log.info("read jmsTracingEventList: {}", events);
 
       enabledEvents.clear();
       for (String event : events.split(",")) {
@@ -109,7 +111,9 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) {
             .getProperties()
             .getProperty("jmsTracingLevel", defaultTraceLevel.toString());
     try {
-      return TraceLevel.valueOf(level.trim().toUpperCase());
+      TraceLevel traceLevel = TraceLevel.valueOf(level.trim().toUpperCase());
+      log.info("Using tracing level: {}", traceLevel);
+      return traceLevel;
     } catch (IllegalArgumentException e) {
       log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel);
       return defaultTraceLevel;
@@ -268,6 +272,17 @@ private static CompletableFuture<TraceLevel> readTraceLevelForTopicAsync(
         });
   }
 
+  @NotNull
+  private static String formatMessageId(MessageIdData x) {
+    String msgId = x.getLedgerId() + ":" + x.getEntryId();
+    if (x.hasBatchIndex()) {
+      msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
+    } else if (x.getAckSetsCount() > 0) {
+      msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
+    }
+    return msgId;
+  }
+
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
 
@@ -277,12 +292,15 @@ public void initialize(PulsarService pulsarService) {
     Properties props = pulsarService.getConfiguration().getProperties();
     if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
       maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
+      log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength);
     }
     if (props.containsKey("jmsTracingTraceSystemTopics")) {
       traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
+      log.info("Setting traceSystemTopics to {}", traceSystemTopics);
     }
     if (props.containsKey("jmsTracingTraceSchema")) {
       traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema"));
+      log.info("Setting traceSchema to {}", traceSchema);
     }
     if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) {
       cacheTraceLevelsDurationSec =
@@ -292,6 +310,7 @@ public void initialize(PulsarService pulsarService) {
             "Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10);
         cacheTraceLevelsDurationSec = 10;
       }
+      log.info("Setting cacheTraceLevelsDurationSec to {}", cacheTraceLevelsDurationSec);
     }
   }
 
@@ -472,12 +491,47 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept
 
     if (traceLevel == TraceLevel.OFF) return;
 
+    traceCommandInternal(command, cnx);
+
+    // todo: cache topics, and check if system topic
+    // without cache:
+
+    //    final String topicName = getCommandTopic(command);
+    //    if (topicName == null) {
+    //      // don't know if a system topic, trace it
+    //      traceCommandInternal(command, cnx);
+    //      return;
+    //    }
+    //
+    //    cnx.getBrokerService()
+    //        .getTopicIfExists(topicName)
+    //        .whenComplete(
+    //            (topic, ex) -> {
+    //              if (ex != null) {
+    //                log.error("Error getting topic {} to trace command {}", topicName, command,
+    // ex);
+    //                return;
+    //              }
+    //
+    //              // skip system topics if needed
+    //              if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic())
+    // return;
+    //
+    //              traceCommandInternal(command, cnx);
+    //            });
+  }
+
+  private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) {
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+
+    traceDetails.put("authMethod", cnx.getAuthMethod());
+    traceDetails.put("authRole", cnx.getAuthRole());
+    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
+    traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
 
     if (command.hasType()) {
-      traceDetails.put("type", command.getType().name());
-      traceDetails.put("command", getCommandDetails(command));
+      traceDetails.put("command", command.getType().name());
+      traceDetails.put("parameters", getCommandDetails(command));
     } else {
       traceDetails.put("type", "unknown/null");
     }
@@ -638,17 +692,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
   }
 
-  @NotNull
-  private static String formatMessageId(MessageIdData x) {
-    String msgId = x.getLedgerId() + ":" + x.getEntryId();
-    if (x.hasBatchIndex()) {
-      msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")";
-    } else if (x.getAckSetsCount() > 0) {
-      msgId += " (ackSetCnt " + x.getAckSetsCount() + ")";
-    }
-    return msgId;
-  }
-
   /* ***************************
    **  Transaction events
    ******************************/
@@ -679,18 +722,38 @@ public void txnEnded(String txnID, long txnAction) {
    **  Servlet events
    ******************************/
 
-  public void onWebserviceRequest(ServletRequest request)
-      throws IOException, ServletException, InterceptException {
-    //    if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
-    //      log.info("onWebserviceRequest: Tracing servlet requests not supported");
-    //    }
+  public void onWebserviceRequest(ServletRequest request) {
+    // skipping, it is the same as onWebserviceResponse
+    // but without response status.
   }
 
-  public void onWebserviceResponse(ServletRequest request, ServletResponse response)
-      throws IOException, ServletException {
-    //    if (getEnabledEvents(???).contains(EventReasons.SERVLET)) {
-    //      log.info("onWebserviceResponse: Tracing servlet requests not supported");
-    //    }
+  public void onWebserviceResponse(ServletRequest request, ServletResponse response) {
+    if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return;
+    if (traceLevel == TraceLevel.OFF) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+
+    traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost()));
+    traceDetails.put("protocol", request.getProtocol());
+    traceDetails.put("scheme", request.getScheme());
+
+    try {
+      HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true);
+      traceDetails.put("method", req.getMethod());
+      traceDetails.put("uri", req.getRequestURI());
+      if (req.getQueryString() != null) {
+        traceDetails.put("queryString", req.getQueryString());
+      }
+      traceDetails.put("authType", req.getAuthType());
+      traceDetails.put("remoteUser", req.getRemoteUser());
+
+      HttpServletResponse resp = (HttpServletResponse) response;
+      traceDetails.put("status", resp.getStatus());
+    } catch (Throwable t) {
+      log.error("Error getting request details", t);
+    }
+
+    trace(EventReasons.SERVLET, "WebService response", traceDetails);
   }
 
   // not needed
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index f0317a0c..136ad5fd 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -360,6 +360,82 @@ private static void populateCommandDetails(
     }
   }
 
+  public static String getCommandTopic(BaseCommand command) {
+    if (command == null) {
+      return null;
+    }
+
+    if (!command.hasType()) {
+      return null;
+    }
+
+    // Currently not doing transitive topic resolution by e.g. topic pattern
+    // or by producerId/consumerId to the topic they are using.
+    // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by
+    // requestId.
+    switch (command.getType()) {
+      case SUBSCRIBE:
+        if (command.getSubscribe().hasTopic()) {
+          return command.getSubscribe().getTopic();
+        }
+        break;
+      case PRODUCER:
+        if (command.getProducer().hasTopic()) {
+          return command.getProducer().getTopic();
+        }
+        break;
+      case PARTITIONED_METADATA:
+        if (command.getPartitionMetadata().hasTopic()) {
+          return command.getPartitionMetadata().getTopic();
+        }
+        break;
+      case LOOKUP:
+        if (command.getLookupTopic().hasTopic()) {
+          return command.getLookupTopic().getTopic();
+        }
+        break;
+      case GET_SCHEMA:
+        if (command.getGetSchema().hasTopic()) {
+          return command.getGetSchema().getTopic();
+        }
+        break;
+      case GET_OR_CREATE_SCHEMA:
+        if (command.getGetOrCreateSchema().hasTopic()) {
+          return command.getGetOrCreateSchema().getTopic();
+        }
+        break;
+      case ADD_SUBSCRIPTION_TO_TXN:
+        if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0
+            && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) {
+          Optional<org.apache.pulsar.common.api.proto.Subscription> subscription =
+              command
+                  .getAddSubscriptionToTxn()
+                  .getSubscriptionsList()
+                  .stream()
+                  .filter(sub -> sub.hasTopic())
+                  .findFirst();
+          if (subscription.isPresent()) {
+            return subscription.get().getTopic();
+          }
+        }
+        break;
+      case END_TXN_ON_PARTITION:
+        if (command.getEndTxnOnPartition().hasTopic()) {
+          return command.getEndTxnOnPartition().getTopic();
+        }
+        break;
+      case END_TXN_ON_SUBSCRIPTION:
+        if (command.getEndTxnOnSubscription().hasSubscription()
+            && command.getEndTxnOnSubscription().getSubscription().hasTopic()) {
+          return command.getEndTxnOnSubscription().getSubscription().getTopic();
+        }
+        break;
+      default:
+        return null;
+    }
+    return null;
+  }
+
   private static final Set<String> skipTraceFields =
       Sets.newHashSet(
           "authdata",

From 70a82888827f8e98a16a1bed682c0dc70a83ba7e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 8 May 2024 14:24:48 -0700
Subject: [PATCH 27/29] jmsTracingLevel as a master off switch; reworked
 categories, renamed config param

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 266 +++++------
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 440 ++----------------
 .../pulsar/jms/tracing/TracingUtilsTest.java  |  12 +-
 3 files changed, 172 insertions(+), 546 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index cc34cf57..dafa8ee0 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -15,9 +15,9 @@
  */
 package com.datastax.oss.pulsar.jms.tracing;
 
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventCategory;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventSubCategory;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails;
@@ -34,7 +34,6 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import io.netty.buffer.ByteBuf;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Properties;
@@ -63,7 +62,7 @@
 import org.apache.pulsar.common.api.proto.CommandAck;
 import org.apache.pulsar.common.api.proto.MessageIdData;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
-import org.apache.pulsar.common.intercept.InterceptException;
+import org.apache.pulsar.common.api.proto.TxnAction;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
@@ -75,7 +74,7 @@ public class BrokerTracing implements BrokerInterceptor {
 
   private static final TraceLevel defaultTraceLevel = TraceLevel.OFF;
 
-  private final Set<EventReasons> jmsTracingEventList = new HashSet<>();
+  private final Set<EventCategory> jmsTracingEventCategory = new HashSet<>();
   private TraceLevel traceLevel = defaultTraceLevel;
   private int maxBinaryDataLength = 256;
   private int cacheTraceLevelsDurationSec = 10;
@@ -83,23 +82,30 @@ public class BrokerTracing implements BrokerInterceptor {
   private boolean traceSchema = false;
 
   private static void loadEnabledEvents(
-      PulsarService pulsarService, Set<EventReasons> enabledEvents) {
+      PulsarService pulsarService, Set<EventCategory> enabledEvents, TraceLevel traceLevel) {
+
+    if (traceLevel == TraceLevel.OFF) {
+      log.info("Tracing is disabled. jmsTracingEventCategory is ignored.");
+      enabledEvents.clear();
+      return;
+    }
+
     Properties props = pulsarService.getConfiguration().getProperties();
-    if (props.containsKey("jmsTracingEventList")) {
-      String events = props.getProperty("jmsTracingEventList", "");
-      log.info("read jmsTracingEventList: {}", events);
+    if (props.containsKey("jmsTracingEventCategory")) {
+      String events = props.getProperty("jmsTracingEventCategory", "");
+      log.info("read jmsTracingEventCategory: {}", events);
 
       enabledEvents.clear();
       for (String event : events.split(",")) {
         try {
-          enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase()));
+          enabledEvents.add(EventCategory.valueOf(event.trim().toUpperCase()));
         } catch (IllegalArgumentException e) {
           log.error("Invalid event: {}. Skipping", event);
         }
       }
     } else {
-      log.warn("jmsTracingEventList not set. Using all available.");
-      enabledEvents.addAll(Arrays.asList(EventReasons.values()));
+      log.warn("jmsTracingEventCategory is not set. Using ADMIN, CONN.");
+      enabledEvents.add(EventCategory.CONN);
     }
   }
 
@@ -286,9 +292,10 @@ private static String formatMessageId(MessageIdData x) {
   public void initialize(PulsarService pulsarService) {
     log.info("Initializing BrokerTracing");
 
-    loadEnabledEvents(pulsarService, jmsTracingEventList);
     traceLevel = getTraceLevel(pulsarService);
 
+    loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel);
+
     Properties props = pulsarService.getConfiguration().getProperties();
     if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
       maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
@@ -393,41 +400,49 @@ private static void addMinimumConsumerSubscriptionDetails(
   }
 
   /* ***************************
-   **  Administrative events
+   **  Connection events
    ******************************/
 
   public void onConnectionCreated(ServerCnx cnx) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+    traceDetails.put("serverCnx", getConnectionDetails(cnx));
+    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
-    if (traceLevel == TraceLevel.OFF) return;
+    trace(EventCategory.CONN, EventSubCategory.CREATED, traceDetails);
+  }
+
+  public void onConnectionClosed(ServerCnx cnx) {
+    if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("serverCnx", getConnectionDetails(cnx));
     traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
-    trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails);
+    trace(EventCategory.CONN, EventSubCategory.CLOSED, traceDetails);
   }
 
+  /* **************************
+   * Producer connection events
+   *****************************/
+
   public void producerCreated(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return;
     if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.OFF) return;
-
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
     traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
-    trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails);
+    trace(EventCategory.PROD, EventSubCategory.CREATED, traceDetails);
   }
 
   public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String> metadata) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return;
     if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.OFF) return;
-
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("producer", getProducerDetails(producer, traceSchema));
     traceDetails.put("metadata", metadata);
@@ -441,30 +456,30 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
     // no message count in stats? stats.getCount() is not it
 
-    trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails);
+    trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails);
   }
 
+  /* **************************
+   * Consumer connection events
+   *****************************/
+
   public void consumerCreated(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return;
     if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.OFF) return;
-
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
     traceDetails.put("metadata", metadata);
     traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
-    trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails);
+    trace(EventCategory.CONS, EventSubCategory.CREATED, traceDetails);
   }
 
   public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String> metadata) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return;
     if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return;
 
-    if (traceLevel == TraceLevel.OFF) return;
-
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("consumer", getConsumerDetails(consumer));
     traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription()));
@@ -483,102 +498,16 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     traceDetails.put("messageAckRate", stats.getMessageAckRate());
     traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
 
-    trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails);
-  }
-
-  public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException {
-    if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return;
-
-    if (traceLevel == TraceLevel.OFF) return;
-
-    traceCommandInternal(command, cnx);
-
-    // todo: cache topics, and check if system topic
-    // without cache:
-
-    //    final String topicName = getCommandTopic(command);
-    //    if (topicName == null) {
-    //      // don't know if a system topic, trace it
-    //      traceCommandInternal(command, cnx);
-    //      return;
-    //    }
-    //
-    //    cnx.getBrokerService()
-    //        .getTopicIfExists(topicName)
-    //        .whenComplete(
-    //            (topic, ex) -> {
-    //              if (ex != null) {
-    //                log.error("Error getting topic {} to trace command {}", topicName, command,
-    // ex);
-    //                return;
-    //              }
-    //
-    //              // skip system topics if needed
-    //              if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic())
-    // return;
-    //
-    //              traceCommandInternal(command, cnx);
-    //            });
-  }
-
-  private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) {
-    Map<String, Object> traceDetails = new TreeMap<>();
-
-    traceDetails.put("authMethod", cnx.getAuthMethod());
-    traceDetails.put("authRole", cnx.getAuthRole());
-    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
-    traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
-
-    if (command.hasType()) {
-      traceDetails.put("command", command.getType().name());
-      traceDetails.put("parameters", getCommandDetails(command));
-    } else {
-      traceDetails.put("type", "unknown/null");
-    }
-
-    trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails);
-  }
-
-  public void onConnectionClosed(ServerCnx cnx) {
-    if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return;
-
-    if (traceLevel == TraceLevel.OFF) return;
-
-    Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("serverCnx", getConnectionDetails(cnx));
-    traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
-
-    trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails);
+    trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails);
   }
 
   /* ***************************
    **  Message events
    ******************************/
 
-  public void beforeSendMessage(
-      Subscription subscription,
-      Entry entry,
-      long[] ackSet,
-      MessageMetadata msgMetadata,
-      Consumer consumer) {
-    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
-
-    TraceLevel level = getTracingLevel(subscription);
-    if (level == TraceLevel.OFF) return;
-
-    Map<String, Object> traceDetails = new TreeMap<>();
-
-    addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
-
-    traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
-
-    trace(EventReasons.MESSAGE, "read", traceDetails);
-  }
-
   public void onMessagePublish(
       Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) {
-
-    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
 
     TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.OFF) return;
@@ -594,7 +523,7 @@ public void onMessagePublish(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "received", traceDetails);
+    trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
   }
 
   public void messageProduced(
@@ -604,7 +533,7 @@ public void messageProduced(
       long ledgerId,
       long entryId,
       Topic.PublishContext publishContext) {
-    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
 
     TraceLevel level = getTracingLevel(producer);
     if (level == TraceLevel.OFF) return;
@@ -615,12 +544,32 @@ public void messageProduced(
     traceDetails.put("publishContext", getPublishContextDetails(publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
     traceDetails.put("startTimeNs", startTimeNs);
-    trace(EventReasons.MESSAGE, "stored", traceDetails);
+    trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails);
+  }
+
+  public void beforeSendMessage(
+      Subscription subscription,
+      Entry entry,
+      long[] ackSet,
+      MessageMetadata msgMetadata,
+      Consumer consumer) {
+    if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
+
+    TraceLevel level = getTracingLevel(subscription);
+    if (level == TraceLevel.OFF) return;
+
+    Map<String, Object> traceDetails = new TreeMap<>();
+
+    addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
+
+    traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+
+    trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
   }
 
   public void messageDispatched(
       ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) {
-    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
 
     TraceLevel level = getTracingLevel(consumer);
     if (level == TraceLevel.OFF) return;
@@ -634,15 +583,17 @@ public void messageDispatched(
         "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
     traceDetails.put("payload", headersAndPayloadDetails);
 
-    trace(EventReasons.MESSAGE, "dispatched", traceDetails);
+    trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
   }
 
   public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
-    if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return;
 
     TraceLevel level = getTracingLevel(consumer);
     if (consumer != null && level == TraceLevel.OFF) return;
 
+    EventSubCategory subcategory = EventSubCategory.ACKED;
+
     Map<String, Object> traceDetails = new TreeMap<>();
 
     addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
@@ -650,6 +601,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     if (consumer == null) {
       // ack with empty consumer == message filtered by JMSFilter
       traceDetails.put("reason", "filtered by JMSFilter");
+      subcategory = EventSubCategory.FILTERED;
     } else {
       // todo: am I right that unacked/nacked messages never go through broker interceptor?
       // in this case we need consumer interceptor to track nacks
@@ -662,7 +614,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
       traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
       traceDetails.put("subscriptionType", sub.getType().name());
     }
-
     Map<String, Object> ackDetails = new TreeMap<>();
     if (ackCmd.hasAckType()) {
       ackDetails.put("type", ackCmd.getAckType().name());
@@ -689,7 +640,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
 
     traceDetails.put("ack", ackDetails);
 
-    trace(EventReasons.MESSAGE, "acknowledged", traceDetails);
+    trace(EventCategory.MSG, subcategory, traceDetails);
   }
 
   /* ***************************
@@ -697,39 +648,51 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
    ******************************/
 
   public void txnOpened(long tcId, String txnID) {
-    if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
-    if (traceLevel == TraceLevel.OFF) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.TX)) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
-    traceDetails.put("tcId", tcId);
+    traceDetails.put("tcId", tcId); // transaction coordinator id
     traceDetails.put("txnID", txnID);
 
-    trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails);
+    trace(EventCategory.TX, EventSubCategory.OPENED, traceDetails);
   }
 
   public void txnEnded(String txnID, long txnAction) {
-    if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return;
-    if (traceLevel == TraceLevel.OFF) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.TX)) return;
 
+    final EventSubCategory subcategory;
     Map<String, Object> traceDetails = new TreeMap<>();
     traceDetails.put("txnID", txnID);
-    traceDetails.put("txnAction", txnAction);
 
-    trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails);
+    TxnAction action = TxnAction.valueOf((int) txnAction);
+    if (action == null) {
+      subcategory = EventSubCategory.CLOSED;
+      traceDetails.put("txnAction", "unknown action code " + txnAction);
+    } else {
+      traceDetails.put("txnAction", action.name());
+      switch (action) {
+        case COMMIT:
+          subcategory = EventSubCategory.COMMITTED;
+          break;
+        case ABORT:
+          subcategory = EventSubCategory.ABORTED;
+          break;
+        default:
+          subcategory = EventSubCategory.CLOSED;
+          traceDetails.put("txnAction", "unknown action code " + txnAction + " " + action.name());
+          break;
+      }
+    }
+
+    trace(EventCategory.TX, subcategory, traceDetails);
   }
 
   /* ***************************
    **  Servlet events
    ******************************/
 
-  public void onWebserviceRequest(ServletRequest request) {
-    // skipping, it is the same as onWebserviceResponse
-    // but without response status.
-  }
-
   public void onWebserviceResponse(ServletRequest request, ServletResponse response) {
-    if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return;
-    if (traceLevel == TraceLevel.OFF) return;
+    if (!jmsTracingEventCategory.contains(EventCategory.REST)) return;
 
     Map<String, Object> traceDetails = new TreeMap<>();
 
@@ -737,6 +700,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
     traceDetails.put("protocol", request.getProtocol());
     traceDetails.put("scheme", request.getScheme());
 
+    // todo: log POST payload?
     try {
       HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true);
       traceDetails.put("method", req.getMethod());
@@ -753,9 +717,21 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
       log.error("Error getting request details", t);
     }
 
-    trace(EventReasons.SERVLET, "WebService response", traceDetails);
+    trace(EventCategory.REST, EventSubCategory.CALLED, traceDetails);
+  }
+
+  /* ***************************
+   **  Skipped
+   ******************************/
+
+  public void onPulsarCommand(BaseCommand command, ServerCnx cnx) {
+    // skipping, output is not useful
+  }
+
+  public void onWebserviceRequest(ServletRequest request) {
+    // skipping, it is the same as onWebserviceResponse
+    // but without response status.
   }
 
-  // not needed
   // public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain)
 }
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 136ad5fd..f8caf8ef 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -21,18 +21,13 @@
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Sets;
 import io.netty.buffer.ByteBuf;
-import java.lang.reflect.Method;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -45,7 +40,6 @@
 import org.apache.pulsar.broker.service.ServerCnx;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
-import org.apache.pulsar.common.api.proto.BaseCommand;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
@@ -56,33 +50,53 @@
 @Slf4j
 public class TracingUtils {
 
-  public enum EventReasons {
-    ADMINISTRATIVE,
-    COMMANDS,
-    MESSAGE,
-    TRANSACTION,
-    SERVLET,
+  public enum EventCategory {
+    CONN, // connection creation, closure,
+    PROD, // producer creation, closure,
+    CONS, // consumer creation, closure,
+    TX, // (transaction creation, commit, rollback,etc),
+    MSG, // (message level send,dispatch,ack,expire,acktimeout, negative ack, etc),
+    REST, // (rest api calls),
+  }
+
+  public enum EventSubCategory {
+    CREATED,
+    CLOSED,
+
+    PRODUCED,
+    STORED,
+
+    READ,
+    DISPATCHED,
+    ACKED,
+    FILTERED,
+
+    OPENED,
+    COMMITTED,
+    ABORTED,
+
+    CALLED,
   }
 
   @FunctionalInterface
   public interface Tracer {
-    void trace(EventReasons reason, String message);
+    void trace(EventCategory category, String message);
   }
 
   public static class Slf4jTracer implements Tracer {
-    private static final Map<EventReasons, org.slf4j.Logger> traceLoggers = new HashMap<>();
+    private static final Map<EventCategory, org.slf4j.Logger> traceLoggers = new HashMap<>();
 
     static {
-      for (EventReasons reason : EventReasons.values()) {
+      for (EventCategory category : EventCategory.values()) {
         traceLoggers.put(
-            reason,
-            org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase()));
+            category,
+            org.slf4j.LoggerFactory.getLogger("jms-tracing-" + category.name().toLowerCase()));
       }
     }
 
     @Override
-    public void trace(EventReasons reason, String message) {
-      traceLoggers.get(reason).info(message);
+    public void trace(EventCategory category, String message) {
+      traceLoggers.get(category).info(message);
     }
   }
 
@@ -136,397 +150,33 @@ public static String hostNameOf(String clientAddress) {
     }
   }
 
-  public static void trace(EventReasons reason, String message, Map<String, Object> traceDetails) {
-    trace(SLF4J_TRACER, reason, message, traceDetails);
+  public static void trace(
+      EventCategory category, EventSubCategory subCategory, Map<String, Object> traceDetails) {
+    trace(SLF4J_TRACER, category, subCategory, traceDetails);
   }
 
   public static void trace(
-      Tracer tracer, EventReasons reason, String message, Map<String, Object> traceDetails) {
+      Tracer tracer,
+      EventCategory category,
+      EventSubCategory subCategory,
+      Map<String, Object> traceDetails) {
     Map<String, Object> trace = new TreeMap<>();
-    trace.put("eventType", message);
+    trace.put("event", category + "_" + subCategory);
     trace.put("traceDetails", traceDetails);
 
     try {
       String loggableJsonString = mapper.writeValueAsString(trace);
-      tracer.trace(reason, loggableJsonString);
+      tracer.trace(category, loggableJsonString);
     } catch (JsonProcessingException e) {
       log.error(
-          "Failed to serialize trace event type '{}' as json, traceDetails: {}",
-          message,
+          "Failed to serialize trace event '{}_{}' as json, traceDetails: {}",
+          category,
+          subCategory,
           traceDetails,
           e);
     }
   }
 
-  public static Map<String, Object> getCommandDetails(BaseCommand command) {
-    if (command == null) {
-      return null;
-    }
-
-    Map<String, Object> details = new TreeMap<>();
-    populateCommandDetails(command, details);
-    return details;
-  }
-
-  private static void populateCommandDetails(
-      BaseCommand command, Map<String, Object> traceDetails) {
-    if (command == null) {
-      return;
-    }
-
-    if (!command.hasType()) {
-      return;
-    }
-
-    // trace all params otherwise
-    switch (command.getType()) {
-      case CONNECT:
-        populateByReflection(command.getConnect(), traceDetails);
-        break;
-      case CONNECTED:
-        populateByReflection(command.getConnected(), traceDetails);
-        break;
-      case SUBSCRIBE:
-        populateByReflection(command.getSubscribe(), traceDetails);
-        break;
-      case PRODUCER:
-        populateByReflection(command.getProducer(), traceDetails);
-        break;
-      case SEND:
-        populateByReflection(command.getSend(), traceDetails);
-        break;
-      case SEND_RECEIPT:
-        populateByReflection(command.getSendReceipt(), traceDetails);
-        break;
-      case SEND_ERROR:
-        populateByReflection(command.getSendError(), traceDetails);
-        break;
-      case MESSAGE:
-        populateByReflection(command.getMessage(), traceDetails);
-        break;
-      case ACK:
-        populateByReflection(command.getAck(), traceDetails);
-        break;
-      case FLOW:
-        populateByReflection(command.getFlow(), traceDetails);
-        break;
-      case UNSUBSCRIBE:
-        populateByReflection(command.getUnsubscribe(), traceDetails);
-        break;
-      case SUCCESS:
-        populateByReflection(command.getSuccess(), traceDetails);
-        break;
-      case ERROR:
-        populateByReflection(command.getError(), traceDetails);
-        break;
-      case CLOSE_PRODUCER:
-        populateByReflection(command.getCloseProducer(), traceDetails);
-        break;
-      case CLOSE_CONSUMER:
-        populateByReflection(command.getCloseConsumer(), traceDetails);
-        break;
-      case PRODUCER_SUCCESS:
-        populateByReflection(command.getProducerSuccess(), traceDetails);
-        break;
-      case PING:
-        populateByReflection(command.getPing(), traceDetails);
-        break;
-      case PONG:
-        populateByReflection(command.getPong(), traceDetails);
-        break;
-      case REDELIVER_UNACKNOWLEDGED_MESSAGES:
-        populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails);
-        break;
-      case PARTITIONED_METADATA:
-        populateByReflection(command.getPartitionMetadata(), traceDetails);
-        break;
-      case PARTITIONED_METADATA_RESPONSE:
-        populateByReflection(command.getPartitionMetadataResponse(), traceDetails);
-        break;
-      case LOOKUP:
-        populateByReflection(command.getLookupTopic(), traceDetails);
-        break;
-      case LOOKUP_RESPONSE:
-        populateByReflection(command.getLookupTopicResponse(), traceDetails);
-        break;
-      case CONSUMER_STATS:
-        populateByReflection(command.getConsumerStats(), traceDetails);
-        break;
-      case CONSUMER_STATS_RESPONSE:
-        populateByReflection(command.getConsumerStatsResponse(), traceDetails);
-        break;
-      case REACHED_END_OF_TOPIC:
-        populateByReflection(command.getReachedEndOfTopic(), traceDetails);
-        break;
-      case SEEK:
-        populateByReflection(command.getSeek(), traceDetails);
-        break;
-      case GET_LAST_MESSAGE_ID:
-        populateByReflection(command.getGetLastMessageId(), traceDetails);
-        break;
-      case GET_LAST_MESSAGE_ID_RESPONSE:
-        populateByReflection(command.getGetLastMessageIdResponse(), traceDetails);
-        break;
-      case ACTIVE_CONSUMER_CHANGE:
-        populateByReflection(command.getActiveConsumerChange(), traceDetails);
-        break;
-      case GET_TOPICS_OF_NAMESPACE:
-        populateByReflection(command.getGetTopicsOfNamespace(), traceDetails);
-        break;
-      case GET_TOPICS_OF_NAMESPACE_RESPONSE:
-        populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails);
-        break;
-      case GET_SCHEMA:
-        populateByReflection(command.getGetSchema(), traceDetails);
-        break;
-      case GET_SCHEMA_RESPONSE:
-        populateByReflection(command.getGetSchemaResponse(), traceDetails);
-        break;
-      case AUTH_CHALLENGE:
-        populateByReflection(command.getAuthChallenge(), traceDetails);
-        break;
-      case AUTH_RESPONSE:
-        populateByReflection(command.getAuthResponse(), traceDetails);
-        break;
-      case ACK_RESPONSE:
-        populateByReflection(command.getAckResponse(), traceDetails);
-        break;
-      case GET_OR_CREATE_SCHEMA:
-        populateByReflection(command.getGetOrCreateSchema(), traceDetails);
-        break;
-      case GET_OR_CREATE_SCHEMA_RESPONSE:
-        populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails);
-        break;
-      case NEW_TXN:
-        populateByReflection(command.getNewTxn(), traceDetails);
-        break;
-      case NEW_TXN_RESPONSE:
-        populateByReflection(command.getNewTxnResponse(), traceDetails);
-        break;
-      case ADD_PARTITION_TO_TXN:
-        populateByReflection(command.getAddPartitionToTxn(), traceDetails);
-        break;
-      case ADD_PARTITION_TO_TXN_RESPONSE:
-        populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails);
-        break;
-      case ADD_SUBSCRIPTION_TO_TXN:
-        populateByReflection(command.getAddSubscriptionToTxn(), traceDetails);
-        break;
-      case ADD_SUBSCRIPTION_TO_TXN_RESPONSE:
-        populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails);
-        break;
-      case END_TXN:
-        populateByReflection(command.getEndTxn(), traceDetails);
-        break;
-      case END_TXN_RESPONSE:
-        populateByReflection(command.getEndTxnResponse(), traceDetails);
-        break;
-      case END_TXN_ON_PARTITION:
-        populateByReflection(command.getEndTxnOnPartition(), traceDetails);
-        break;
-      case END_TXN_ON_PARTITION_RESPONSE:
-        populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails);
-        break;
-      case END_TXN_ON_SUBSCRIPTION:
-        populateByReflection(command.getEndTxnOnSubscription(), traceDetails);
-        break;
-      case END_TXN_ON_SUBSCRIPTION_RESPONSE:
-        populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails);
-        break;
-      case TC_CLIENT_CONNECT_REQUEST:
-        populateByReflection(command.getTcClientConnectRequest(), traceDetails);
-        break;
-      case TC_CLIENT_CONNECT_RESPONSE:
-        populateByReflection(command.getTcClientConnectResponse(), traceDetails);
-        break;
-      case WATCH_TOPIC_LIST:
-        populateByReflection(command.getWatchTopicList(), traceDetails);
-        break;
-      case WATCH_TOPIC_LIST_SUCCESS:
-        populateByReflection(command.getWatchTopicListSuccess(), traceDetails);
-        break;
-      case WATCH_TOPIC_UPDATE:
-        populateByReflection(command.getWatchTopicUpdate(), traceDetails);
-        break;
-      case WATCH_TOPIC_LIST_CLOSE:
-        populateByReflection(command.getWatchTopicListClose(), traceDetails);
-        break;
-      case TOPIC_MIGRATED:
-        populateByReflection(command.getTopicMigrated(), traceDetails);
-        break;
-      default:
-        log.error("Unknown command type: {}", command.getType());
-        traceDetails.put("error", "unknownCommandType " + command.getType());
-    }
-  }
-
-  public static String getCommandTopic(BaseCommand command) {
-    if (command == null) {
-      return null;
-    }
-
-    if (!command.hasType()) {
-      return null;
-    }
-
-    // Currently not doing transitive topic resolution by e.g. topic pattern
-    // or by producerId/consumerId to the topic they are using.
-    // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by
-    // requestId.
-    switch (command.getType()) {
-      case SUBSCRIBE:
-        if (command.getSubscribe().hasTopic()) {
-          return command.getSubscribe().getTopic();
-        }
-        break;
-      case PRODUCER:
-        if (command.getProducer().hasTopic()) {
-          return command.getProducer().getTopic();
-        }
-        break;
-      case PARTITIONED_METADATA:
-        if (command.getPartitionMetadata().hasTopic()) {
-          return command.getPartitionMetadata().getTopic();
-        }
-        break;
-      case LOOKUP:
-        if (command.getLookupTopic().hasTopic()) {
-          return command.getLookupTopic().getTopic();
-        }
-        break;
-      case GET_SCHEMA:
-        if (command.getGetSchema().hasTopic()) {
-          return command.getGetSchema().getTopic();
-        }
-        break;
-      case GET_OR_CREATE_SCHEMA:
-        if (command.getGetOrCreateSchema().hasTopic()) {
-          return command.getGetOrCreateSchema().getTopic();
-        }
-        break;
-      case ADD_SUBSCRIPTION_TO_TXN:
-        if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0
-            && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) {
-          Optional<org.apache.pulsar.common.api.proto.Subscription> subscription =
-              command
-                  .getAddSubscriptionToTxn()
-                  .getSubscriptionsList()
-                  .stream()
-                  .filter(sub -> sub.hasTopic())
-                  .findFirst();
-          if (subscription.isPresent()) {
-            return subscription.get().getTopic();
-          }
-        }
-        break;
-      case END_TXN_ON_PARTITION:
-        if (command.getEndTxnOnPartition().hasTopic()) {
-          return command.getEndTxnOnPartition().getTopic();
-        }
-        break;
-      case END_TXN_ON_SUBSCRIPTION:
-        if (command.getEndTxnOnSubscription().hasSubscription()
-            && command.getEndTxnOnSubscription().getSubscription().hasTopic()) {
-          return command.getEndTxnOnSubscription().getSubscription().getTopic();
-        }
-        break;
-      default:
-        return null;
-    }
-    return null;
-  }
-
-  private static final Set<String> skipTraceFields =
-      Sets.newHashSet(
-          "authdata",
-          "authmethod",
-          "authmethodname",
-          "originalauthdata",
-          "orginalauthmethod",
-          "originalprincipal",
-          "schema");
-
-  private static void populateByReflection(Object command, Map<String, Object> traceDetails) {
-    if (command == null) {
-      return;
-    }
-    if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) {
-      return;
-    }
-
-    Method[] allMethods = command.getClass().getMethods();
-
-    Arrays.stream(allMethods)
-        .filter(
-            method -> {
-              if (!method.getName().startsWith("has")) {
-                return false;
-              }
-              String fieldName = method.getName().substring(3);
-              return !skipTraceFields.contains(fieldName.toLowerCase());
-            })
-        .filter(
-            method -> {
-              try {
-                return (boolean) method.invoke(command);
-              } catch (Exception e) {
-                return false;
-              }
-            })
-        .forEach(
-            method -> {
-              String fieldName = method.getName().substring(3);
-              try {
-                Optional<Method> accessor =
-                    Arrays.stream(allMethods)
-                        .filter(
-                            m ->
-                                m.getName().equals("get" + fieldName)
-                                    || m.getName().equals("is" + fieldName))
-                        .findFirst();
-                if (!accessor.isPresent()) {
-                  log.warn(
-                      "No accessor found for field (but has.. counterpart was found): {} of {}",
-                      fieldName,
-                      command.getClass().getCanonicalName());
-                  return;
-                }
-                Object value = accessor.get().invoke(command);
-
-                if (value == null) return;
-
-                // skip logging of binary data
-                if (value instanceof byte[]
-                    || value instanceof ByteBuf
-                    || value instanceof ByteBuffer) {
-                  final int size;
-                  if (value instanceof byte[]) {
-                    size = ((byte[]) value).length;
-                  } else if (value instanceof ByteBuf) {
-                    size = ((ByteBuf) value).readableBytes();
-                  } else {
-                    size = ((ByteBuffer) value).remaining();
-                  }
-                  traceDetails.put(fieldName + "_size", size);
-                  return;
-                }
-
-                if (value
-                    .getClass()
-                    .getCanonicalName()
-                    .contains("org.apache.pulsar.common.api.proto")) {
-                  Map<String, Object> details = new TreeMap<>();
-                  populateByReflection(value, details);
-                  traceDetails.put(fieldName, details);
-                } else {
-                  traceDetails.put(fieldName, value);
-                }
-              } catch (Exception e) {
-                log.error("Failed to access field: {}", fieldName, e);
-              }
-            });
-  }
-
   public static Map<String, Object> getConnectionDetails(ServerCnx cnx) {
     if (cnx == null) {
       return null;
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 71e01bba..9fa3e0d4 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -32,33 +32,33 @@ class TracingUtilsTest {
   private Tracer mockTracer =
       new Tracer() {
         @Override
-        public void trace(EventReasons reason, String msg) {
+        public void trace(EventCategory reason, String msg) {
           traces.add(msg);
         }
       };
 
-  private static void trace(Tracer mockTracer, String msg, Map<String, Object> traceDetails) {
-    TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails);
+  private static void trace(Tracer mockTracer, Map<String, Object> traceDetails) {
+    TracingUtils.trace(mockTracer, EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
   }
 
   @Test
   void traceTest() {
     traces.clear();
-    trace(mockTracer, "msg", null);
+    trace(mockTracer, null);
     assertEquals(1, traces.size());
     assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
 
     Map<String, Object> map = new TreeMap<>();
 
     traces.clear();
-    trace(mockTracer, "msg", map);
+    trace(mockTracer, map);
     assertEquals(1, traces.size());
     assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
 
     map.put("key1", "value1");
 
     traces.clear();
-    trace(mockTracer, "msg", map);
+    trace(mockTracer, map);
     assertEquals(1, traces.size());
     assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
   }

From a447548495ffd574e2a341e069e1f4678bce04b4 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Wed, 8 May 2024 17:52:28 -0700
Subject: [PATCH 28/29] PAYLOAD trace levele for messages, port for the host,
 tweaks for details

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java |  52 +++---
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 153 +++++++++++-------
 .../pulsar/jms/tracing/TracingUtilsTest.java  |  34 ++--
 3 files changed, 152 insertions(+), 87 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index dafa8ee0..cfcffa90 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -26,7 +26,7 @@
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf;
 import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace;
-import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf;
+import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceMetadataAndPayload;
 
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
@@ -76,7 +76,7 @@ public class BrokerTracing implements BrokerInterceptor {
 
   private final Set<EventCategory> jmsTracingEventCategory = new HashSet<>();
   private TraceLevel traceLevel = defaultTraceLevel;
-  private int maxBinaryDataLength = 256;
+  private int maxPayloadLength = 256;
   private int cacheTraceLevelsDurationSec = 10;
   private boolean traceSystemTopics = false;
   private boolean traceSchema = false;
@@ -297,9 +297,9 @@ public void initialize(PulsarService pulsarService) {
     loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel);
 
     Properties props = pulsarService.getConfiguration().getProperties();
-    if (props.containsKey("jmsTracingMaxBinaryDataLength")) {
-      maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength"));
-      log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength);
+    if (props.containsKey("jmsTracingMaxPayloadLength")) {
+      maxPayloadLength = Integer.parseInt(props.getProperty("jmsTracingMaxPayloadLength"));
+      log.info("Setting maxPayloadLength to {}", maxPayloadLength);
     }
     if (props.containsKey("jmsTracingTraceSystemTopics")) {
       traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics"));
@@ -367,7 +367,9 @@ private static void addMinimumProducerDetails(
     if (producer.getAccessMode() != null) {
       traceDetails.put("accessMode", producer.getAccessMode().name());
     }
-    traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress()));
+    traceDetails.put("clientHost",
+            TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+
     if (producer.getTopic() != null) {
       traceDetails.put(
           "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
@@ -387,7 +389,8 @@ private static void addMinimumConsumerSubscriptionDetails(
     if (consumer != null) {
       traceDetails.put("consumerName", consumer.consumerName());
       traceDetails.put("consumerId", consumer.consumerId());
-      traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress()));
+      traceDetails.put("clientHost",
+              TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
       traceDetails.put("authRole", consumer.cnx().getAuthRole());
     }
 
@@ -516,12 +519,17 @@ public void onMessagePublish(
 
     addMinimumProducerDetails(producer, traceDetails);
 
-    traceDetails.put("publishContext", getPublishContextDetails(publishContext));
+    traceDetails.put("publishContext", getPublishContextDetails(level, publishContext));
 
-    Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
-    traceByteBuf(
-        "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
-    traceDetails.put("payload", headersAndPayloadDetails);
+    if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
+      Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+      traceMetadataAndPayload(
+          "headersAndPayload",
+          headersAndPayload.slice(),
+          headersAndPayloadDetails,
+          maxPayloadLength);
+      traceDetails.put("payload", headersAndPayloadDetails);
+    }
 
     trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
   }
@@ -541,9 +549,8 @@ public void messageProduced(
     Map<String, Object> traceDetails = new TreeMap<>();
     addMinimumProducerDetails(producer, traceDetails);
 
-    traceDetails.put("publishContext", getPublishContextDetails(publishContext));
+    traceDetails.put("publishContext", getPublishContextDetails(level, publishContext));
     traceDetails.put("messageId", ledgerId + ":" + entryId);
-    traceDetails.put("startTimeNs", startTimeNs);
     trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails);
   }
 
@@ -562,7 +569,7 @@ public void beforeSendMessage(
 
     addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
 
-    traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength));
+    traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength));
 
     trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
   }
@@ -578,10 +585,15 @@ public void messageDispatched(
     addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
     traceDetails.put("messageId", ledgerId + ":" + entryId);
 
-    Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
-    traceByteBuf(
-        "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength);
-    traceDetails.put("payload", headersAndPayloadDetails);
+    if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
+      Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
+      traceMetadataAndPayload(
+          "headersAndPayload",
+          headersAndPayload.slice(),
+          headersAndPayloadDetails,
+          maxPayloadLength);
+      traceDetails.put("payload", headersAndPayloadDetails);
+    }
 
     trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
   }
@@ -696,7 +708,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
 
     Map<String, Object> traceDetails = new TreeMap<>();
 
-    traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost()));
+    traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
     traceDetails.put("protocol", request.getProtocol());
     traceDetails.put("scheme", request.getScheme());
 
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index f8caf8ef..4250e867 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -108,8 +108,9 @@ public void trace(EventCategory category, String message) {
           .enable(SerializationFeature.WRITE_NULL_MAP_VALUES);
 
   public enum TraceLevel {
-    OFF,
-    ON
+    OFF, // disabled
+    ON, // enabled without payload tracing
+    PAYLOAD, // enabled with payload tracing
   }
 
   private static final LoadingCache<String, String> ipResolverCache =
@@ -137,13 +138,28 @@ public String load(String clientAddress) {
                 }
               });
 
-  public static String hostNameOf(String clientAddress) {
+  public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) {
+    if (clientAddress == null || clientAddress.isEmpty()
+            || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) {
+      return "unknown/null";
+    }
+
+    try {
+      String port = clientSourceAddressAndPort.split(":")[1];
+      return ipResolverCache.get(clientAddress) + ":" + port;
+    } catch (Throwable t) {
+      log.error("Failed to resolve DNS for {}", clientAddress, t);
+      return clientAddress;
+    }
+  }
+
+  public static String hostNameOf(String clientAddress, int remotePort) {
     if (clientAddress == null || clientAddress.isEmpty()) {
       return "unknown/null";
     }
 
     try {
-      return ipResolverCache.get(clientAddress);
+      return ipResolverCache.get(clientAddress) + ":" + remotePort;
     } catch (Throwable t) {
       log.error("Failed to resolve DNS for {}", clientAddress, t);
       return clientAddress;
@@ -191,9 +207,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
     if (cnx == null) {
       return;
     }
-    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress()));
+    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
     traceDetails.put("authRole", cnx.getAuthRole());
-    traceDetails.put("principal", cnx.getPrincipal());
     traceDetails.put("clientVersion", cnx.getClientVersion());
     traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
     traceDetails.put("authMethod", cnx.getAuthMethod());
@@ -293,7 +308,7 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
 
     traceDetails.put("priorityLevel", consumer.getPriorityLevel());
     traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
-    traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress()));
+    traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
 
     traceDetails.put("metadata", consumer.getMetadata());
     traceDetails.put("unackedMessages", consumer.getUnackedMessages());
@@ -325,7 +340,7 @@ private static void populateProducerDetails(
           "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
     }
 
-    traceDetails.put("clientHost", hostNameOf(producer.getClientAddress()));
+    traceDetails.put("clientHost", hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
 
     traceDetails.put("metadata", producer.getMetadata());
 
@@ -389,18 +404,19 @@ private static void populateMessageMetadataDetails(
     }
   }
 
-  public static Map<String, Object> getEntryDetails(Entry entry, int maxBinaryDataLength) {
+  public static Map<String, Object> getEntryDetails(
+      TraceLevel level, Entry entry, int maxBinaryDataLength) {
     if (entry == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populateEntryDetails(entry, details, maxBinaryDataLength);
+    populateEntryDetails(level, entry, details, maxBinaryDataLength);
     return details;
   }
 
   private static void populateEntryDetails(
-      Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
+      TraceLevel level, Entry entry, Map<String, Object> traceDetails, int maxBinaryDataLength) {
     if (entry == null) {
       return;
     }
@@ -409,75 +425,98 @@ private static void populateEntryDetails(
 
     traceDetails.put("length", entry.getLength());
 
-    traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength);
+    if (TraceLevel.PAYLOAD == level && entry.getDataBuffer() != null) {
+      traceMetadataAndPayload(
+          "payload", entry.getDataBuffer().slice(), traceDetails, maxBinaryDataLength);
+    }
   }
 
-  public static Map<String, Object> getPublishContextDetails(Topic.PublishContext publishContext) {
+  public static Map<String, Object> getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) {
     if (publishContext == null) {
       return null;
     }
 
     Map<String, Object> details = new TreeMap<>();
-    populatePublishContext(publishContext, details);
+    populatePublishContext(level, publishContext, details);
     return details;
   }
 
-  private static void populatePublishContext(
+  private static void populatePublishContext(TraceLevel level,
       Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
-    traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
-    traceDetails.put("isChunked", publishContext.isChunked());
-    traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+    traceDetails.put("sequenceId", publishContext.getSequenceId());
     traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
     traceDetails.put("msgSize", publishContext.getMsgSize());
-    if (publishContext.getOriginalProducerName() != null) {
-      traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
-      traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+
+    if (TraceLevel.PAYLOAD == level) {
+      traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages());
+      traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage());
+      traceDetails.put("isChunked", publishContext.isChunked());
+      if (publishContext.getOriginalProducerName() != null) {
+        traceDetails.put("originalProducerName", publishContext.getOriginalProducerName());
+        traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId());
+      }
     }
-    traceDetails.put("sequenceId", publishContext.getSequenceId());
   }
 
-  public static void traceByteBuf(
-      String key, ByteBuf buf, Map<String, Object> traceDetails, int maxBinaryDataLength) {
-    if (buf == null || maxBinaryDataLength <= 0) return;
+  /** this will release metadataAndPayload */
+  public static void traceMetadataAndPayload(
+      String key,
+      ByteBuf metadataAndPayload,
+      Map<String, Object> traceDetails,
+      int maxPayloadLength) {
+    if (metadataAndPayload == null) return;
+    if (maxPayloadLength <= 0) {
+      metadataAndPayload.release();
+      return;
+    }
     try {
+      // advance readerIndex
+      MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
+
+      // todo: do we need to trace this metadata?
+      populateMessageMetadataDetails(metadata, traceDetails);
+
+      // Decode if needed
+      CompressionCodec codec =
+          CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
+      ByteBuf uncompressedPayload =
+          codec.decode(metadataAndPayload, metadata.getUncompressedSize());
+      traceByteBuf(key, uncompressedPayload, traceDetails, maxPayloadLength);
+    } catch (Throwable t) {
+      log.error("Failed to trace metadataAndPayload", t);
+    } finally {
+      metadataAndPayload.release();
+    }
+  }
 
-      final ByteBuf metadataAndPayload = buf.retainedDuplicate();
-      ByteBuf uncompressedPayload = null;
-      try {
-        // advance readerIndex
-        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
-
-        // todo: do we need to trace this metadata?
-        populateMessageMetadataDetails(metadata, traceDetails);
-
-        // Decode if needed
-        CompressionCodec codec =
-            CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
-        uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
-
-        // todo: does this require additional steps if messages are batched?
-        if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) {
-          String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8);
-          traceDetails.put(key, dataAsString);
-        } else {
-          String dataAsString =
-              uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8);
-          traceDetails.put(key, dataAsString + "...");
-        }
-      } finally {
-        metadataAndPayload.release();
-        if (uncompressedPayload != null) {
-          uncompressedPayload.release();
-        }
+  /** this will release payload */
+  public static void traceByteBuf(
+      String key, ByteBuf payload, Map<String, Object> traceDetails, int maxPayloadLength) {
+    if (payload == null) return;
+
+    if (maxPayloadLength <= 0) {
+      payload.release();
+      return;
+    }
+
+    try {
+      // todo: does this require additional steps if messages are batched?
+      String dataAsString = payload.toString(StandardCharsets.UTF_8);
+      if (dataAsString.length() > maxPayloadLength + 3) {
+        dataAsString = dataAsString.substring(0, maxPayloadLength) + "...";
       }
+      traceDetails.put(key, dataAsString);
     } catch (Throwable t) {
       log.error("Failed to convert ByteBuf to string", t);
-      if (buf.readableBytes() < maxBinaryDataLength + 3) {
-        traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()));
+      if (payload.readableBytes() < maxPayloadLength) {
+        traceDetails.put(key, "0x" + Hex.encodeHexString(payload.nioBuffer()));
       } else {
-        traceDetails.put(
-            key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "...");
+        ByteBuf buf = payload.slice(0, maxPayloadLength / 2);
+        traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()) + "...");
+        buf.release();
       }
+    } finally {
+      payload.release();
     }
   }
 }
diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
index 9fa3e0d4..3a87a5cf 100644
--- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
+++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java
@@ -20,9 +20,11 @@
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.TreeMap;
 import org.junit.jupiter.api.Test;
 
@@ -46,21 +48,22 @@ void traceTest() {
     traces.clear();
     trace(mockTracer, null);
     assertEquals(1, traces.size());
-    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0));
+    assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":null}", traces.get(0));
 
     Map<String, Object> map = new TreeMap<>();
 
     traces.clear();
     trace(mockTracer, map);
     assertEquals(1, traces.size());
-    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0));
+    assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{}}", traces.get(0));
 
     map.put("key1", "value1");
 
     traces.clear();
     trace(mockTracer, map);
     assertEquals(1, traces.size());
-    assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
+    assertEquals(
+        "{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0));
   }
 
   // todo:
@@ -96,29 +99,40 @@ void traceTest() {
   void traceByteBufTest() {
     Map<String, Object> traceDetails = new TreeMap<>();
 
-    int maxBinaryDataLength = 1024;
+    int maxBinaryDataLength = 100;
 
     traceByteBuf("key", null, traceDetails, maxBinaryDataLength);
     assertEquals(0, traceDetails.size());
 
+    Random rand = new Random();
+
     ByteBuf small = Unpooled.buffer(20);
     for (int i = 0; i < 20; i++) {
-      small.writeByte(i);
+      char randomChar = (char) (rand.nextInt(26) + 'a');
+      small.writeByte(randomChar);
     }
+    String smallStr = small.toString(StandardCharsets.UTF_8);
+    assertEquals(1, small.refCnt());
+
     traceByteBuf("key", small, traceDetails, maxBinaryDataLength);
     assertEquals(1, traceDetails.size());
-    assertEquals(42, ((String) traceDetails.get("key")).length());
-    assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key"));
+    assertEquals(20, ((String) traceDetails.get("key")).length());
+    assertEquals(smallStr, traceDetails.get("key"));
+    assertEquals(0, small.refCnt());
 
     ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100);
     for (int i = 0; i < maxBinaryDataLength + 100; i++) {
-      big.writeByte(i);
+      char randomChar = (char) (rand.nextInt(26) + 'a');
+      big.writeByte(randomChar);
     }
+    assertEquals(1, big.refCnt());
+    String bigStr = big.toString(StandardCharsets.UTF_8);
 
     traceDetails.clear();
     traceByteBuf("key", big, traceDetails, maxBinaryDataLength);
     assertEquals(1, traceDetails.size());
-    assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
-    assertTrue(((String) traceDetails.get("key")).startsWith("0x000102"));
+    assertEquals(maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length());
+    assertEquals(bigStr.substring(0, maxBinaryDataLength) + "...", traceDetails.get("key"));
+    assertEquals(0, big.refCnt());
   }
 }

From db0ad0c103f33eeecf12deee53706edfc864bd25 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <andrey.yegorov@datastax.com>
Date: Fri, 10 May 2024 17:51:50 -0700
Subject: [PATCH 29/29] deduped messages, aded stats etc

---
 .../oss/pulsar/jms/tracing/BrokerTracing.java | 95 +++++++++----------
 .../oss/pulsar/jms/tracing/TracingUtils.java  | 33 +++----
 2 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
index cfcffa90..13999322 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java
@@ -367,8 +367,10 @@ private static void addMinimumProducerDetails(
     if (producer.getAccessMode() != null) {
       traceDetails.put("accessMode", producer.getAccessMode().name());
     }
-    traceDetails.put("clientHost",
-            TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+    traceDetails.put(
+        "clientHost",
+        TracingUtils.hostNameOf(
+            producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
 
     if (producer.getTopic() != null) {
       traceDetails.put(
@@ -389,8 +391,10 @@ private static void addMinimumConsumerSubscriptionDetails(
     if (consumer != null) {
       traceDetails.put("consumerName", consumer.consumerName());
       traceDetails.put("consumerId", consumer.consumerId());
-      traceDetails.put("clientHost",
-              TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
+      traceDetails.put(
+          "clientHost",
+          TracingUtils.hostNameOf(
+              consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
       traceDetails.put("authRole", consumer.cnx().getAuthRole());
     }
 
@@ -451,13 +455,16 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map<String, String>
     traceDetails.put("metadata", metadata);
     traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
+    Map<String, Object> statsTrace = new TreeMap<>();
     PublisherStatsImpl stats = producer.getStats();
-    traceDetails.put("connectedSince", stats.getConnectedSince());
-    traceDetails.put("closedAt", DateFormatter.now());
-    traceDetails.put("averageMsgSize", stats.getAverageMsgSize());
-    traceDetails.put("msgRateIn", stats.getMsgRateIn());
-    traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn());
+
+    statsTrace.put("connectedSince", stats.getConnectedSince());
+    statsTrace.put("closedAt", DateFormatter.now());
+    statsTrace.put("averageMsgSize", stats.getAverageMsgSize());
+    statsTrace.put("msgRateIn", stats.getMsgRateIn());
+    statsTrace.put("msgThroughputIn", stats.getMsgThroughputIn());
     // no message count in stats? stats.getCount() is not it
+    traceDetails.put("stats", statsTrace);
 
     trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails);
   }
@@ -490,16 +497,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map<String, String>
     traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl());
 
     ConsumerStatsImpl stats = consumer.getStats();
-    traceDetails.put("connectedSince", stats.getConnectedSince());
-    traceDetails.put("closedAt", DateFormatter.now());
-    traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry());
-    traceDetails.put("msgRateOut", stats.getMsgRateOut());
-    traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut());
-    traceDetails.put("msgOutCounter", stats.getMsgOutCounter());
-    traceDetails.put("bytesOutCounter", stats.getBytesOutCounter());
-    traceDetails.put("unackedMessages", stats.getUnackedMessages());
-    traceDetails.put("messageAckRate", stats.getMessageAckRate());
-    traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver());
+    Map<String, Object> statsTrace = new TreeMap<>();
+    statsTrace.put("connectedSince", stats.getConnectedSince());
+    statsTrace.put("closedAt", DateFormatter.now());
+    statsTrace.put("averageMsgSize", stats.getAvgMessagesPerEntry());
+    statsTrace.put("msgRateOut", stats.getMsgRateOut());
+    statsTrace.put("msgThroughputOut", stats.getMsgThroughputOut());
+    statsTrace.put("msgOutCounter", stats.getMsgOutCounter());
+    statsTrace.put("bytesOutCounter", stats.getBytesOutCounter());
+    statsTrace.put("unackedMessages", stats.getUnackedMessages());
+    statsTrace.put("messageAckRate", stats.getMessageAckRate());
+    statsTrace.put("msgRateRedeliver", stats.getMsgRateRedeliver());
+    statsTrace.put("readPositionWhenJoining", stats.getReadPositionWhenJoining());
+    Subscription sub = consumer.getSubscription();
+    if (sub != null) {
+      statsTrace.put("subscriptionApproxBacklog", sub.getNumberOfEntriesInBacklog(false));
+      statsTrace.put("subscriptionMsgRateExpired", sub.getExpiredMessageRate());
+    }
+    traceDetails.put("stats", statsTrace);
 
     trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails);
   }
@@ -524,11 +539,11 @@ public void onMessagePublish(
     if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
       Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
       traceMetadataAndPayload(
-          "headersAndPayload",
-          headersAndPayload.slice(),
+          "payload",
+          headersAndPayload.retainedDuplicate(),
           headersAndPayloadDetails,
           maxPayloadLength);
-      traceDetails.put("payload", headersAndPayloadDetails);
+      traceDetails.put("headersAndPayload", headersAndPayloadDetails);
     }
 
     trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails);
@@ -569,7 +584,9 @@ public void beforeSendMessage(
 
     addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails);
 
-    traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength));
+    traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
+
+    traceDetails.put("headersAndPayload", getEntryDetails(level, entry, maxPayloadLength));
 
     trace(EventCategory.MSG, EventSubCategory.READ, traceDetails);
   }
@@ -588,11 +605,11 @@ public void messageDispatched(
     if (TraceLevel.PAYLOAD == level && headersAndPayload != null) {
       Map<String, Object> headersAndPayloadDetails = new TreeMap<>();
       traceMetadataAndPayload(
-          "headersAndPayload",
-          headersAndPayload.slice(),
+          "payload",
+          headersAndPayload.retainedDuplicate(),
           headersAndPayloadDetails,
           maxPayloadLength);
-      traceDetails.put("payload", headersAndPayloadDetails);
+      traceDetails.put("headersAndPayload", headersAndPayloadDetails);
     }
 
     trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails);
@@ -604,35 +621,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
     TraceLevel level = getTracingLevel(consumer);
     if (consumer != null && level == TraceLevel.OFF) return;
 
-    EventSubCategory subcategory = EventSubCategory.ACKED;
-
     Map<String, Object> traceDetails = new TreeMap<>();
 
     addMinimumConsumerSubscriptionDetails(consumer, traceDetails);
 
-    if (consumer == null) {
-      // ack with empty consumer == message filtered by JMSFilter
-      traceDetails.put("reason", "filtered by JMSFilter");
-      subcategory = EventSubCategory.FILTERED;
-    } else {
-      // todo: am I right that unacked/nacked messages never go through broker interceptor?
-      // in this case we need consumer interceptor to track nacks
-      traceDetails.put("reason", "acked");
-    }
-
-    if (consumer != null && consumer.getSubscription() != null) {
-      Subscription sub = consumer.getSubscription();
-      traceDetails.put("subscriptionName", sub.getName());
-      traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-      traceDetails.put("subscriptionType", sub.getType().name());
-    }
     Map<String, Object> ackDetails = new TreeMap<>();
     if (ackCmd.hasAckType()) {
       ackDetails.put("type", ackCmd.getAckType().name());
     }
-    if (ackCmd.hasConsumerId()) {
-      ackDetails.put("ackConsumerId", ackCmd.getConsumerId());
-    }
     ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount());
     ackDetails.put(
         "messageIds",
@@ -652,6 +648,9 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) {
 
     traceDetails.put("ack", ackDetails);
 
+    EventSubCategory subcategory =
+        consumer == null ? EventSubCategory.FILTERED : EventSubCategory.ACKED;
+
     trace(EventCategory.MSG, subcategory, traceDetails);
   }
 
@@ -708,7 +707,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons
 
     Map<String, Object> traceDetails = new TreeMap<>();
 
-    traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
+    traceDetails.put("host", hostNameOf(request.getRemoteHost(), request.getRemotePort()));
     traceDetails.put("protocol", request.getProtocol());
     traceDetails.put("scheme", request.getScheme());
 
diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
index 4250e867..c6e6534b 100644
--- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
+++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java
@@ -139,8 +139,10 @@ public String load(String clientAddress) {
               });
 
   public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) {
-    if (clientAddress == null || clientAddress.isEmpty()
-            || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) {
+    if (clientAddress == null
+        || clientAddress.isEmpty()
+        || clientSourceAddressAndPort == null
+        || !clientSourceAddressAndPort.contains(":")) {
       return "unknown/null";
     }
 
@@ -207,7 +209,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map<String, Object>
     if (cnx == null) {
       return;
     }
-    traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
+    traceDetails.put(
+        "clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort()));
     traceDetails.put("authRole", cnx.getAuthRole());
     traceDetails.put("clientVersion", cnx.getClientVersion());
     traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort());
@@ -300,15 +303,12 @@ private static void populateConsumerDetails(Consumer consumer, Map<String, Objec
 
     traceDetails.put("name", consumer.consumerName());
     traceDetails.put("consumerId", consumer.consumerId());
-    Subscription sub = consumer.getSubscription();
-    if (sub != null) {
-      traceDetails.put("subscriptionName", sub.getName());
-      traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName());
-    }
 
     traceDetails.put("priorityLevel", consumer.getPriorityLevel());
     traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name());
-    traceDetails.put("clientHost", hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
+    traceDetails.put(
+        "clientHost",
+        hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort()));
 
     traceDetails.put("metadata", consumer.getMetadata());
     traceDetails.put("unackedMessages", consumer.getUnackedMessages());
@@ -340,7 +340,9 @@ private static void populateProducerDetails(
           "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName());
     }
 
-    traceDetails.put("clientHost", hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
+    traceDetails.put(
+        "clientHost",
+        hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort()));
 
     traceDetails.put("metadata", producer.getMetadata());
 
@@ -421,17 +423,16 @@ private static void populateEntryDetails(
       return;
     }
 
-    traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId());
-
     traceDetails.put("length", entry.getLength());
 
     if (TraceLevel.PAYLOAD == level && entry.getDataBuffer() != null) {
       traceMetadataAndPayload(
-          "payload", entry.getDataBuffer().slice(), traceDetails, maxBinaryDataLength);
+          "payload", entry.getDataBuffer().retainedDuplicate(), traceDetails, maxBinaryDataLength);
     }
   }
 
-  public static Map<String, Object> getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) {
+  public static Map<String, Object> getPublishContextDetails(
+      TraceLevel level, Topic.PublishContext publishContext) {
     if (publishContext == null) {
       return null;
     }
@@ -441,8 +442,8 @@ public static Map<String, Object> getPublishContextDetails(TraceLevel level, Top
     return details;
   }
 
-  private static void populatePublishContext(TraceLevel level,
-      Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
+  private static void populatePublishContext(
+      TraceLevel level, Topic.PublishContext publishContext, Map<String, Object> traceDetails) {
     traceDetails.put("sequenceId", publishContext.getSequenceId());
     traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp());
     traceDetails.put("msgSize", publishContext.getMsgSize());