Missing namespaces metrics #17656
|
Hello. I have the following question: Let’s say I have two topics:
For each of the topics, there is // Scala code with Pulsar Java admin client.
val adminClient = PulsarAdmin.builder().serviceHttpUrl(config.pulsarProxyUrl).build()
val metricsJson = adminClient.brokerStats().getMetrics()Does anyone have an idea why Pulsar 2.10.1 Thank you. |
Replies: 5 comments 1 reply
|
Use |
|
Namespace level metrics are exactly what I want in this case. |
|
For illustration, I have the following namespaces/topics hierarchy:
For all topics listed on the screenshot, there is a running
[
{
"metrics": {
"brk_ml_cache_evictions": 0,
"brk_ml_cache_hits_rate": 0.0,
"brk_ml_cache_hits_throughput": 0.0,
"brk_ml_cache_misses_rate": 0.0,
"brk_ml_cache_misses_throughput": 0.0,
"brk_ml_cache_pool_active_allocations": 0,
"brk_ml_cache_pool_active_allocations_huge": 0,
"brk_ml_cache_pool_active_allocations_normal": 0,
"brk_ml_cache_pool_active_allocations_small": 0,
"brk_ml_cache_pool_allocated": 0,
"brk_ml_cache_pool_used": 0,
"brk_ml_cache_used_size": 0,
"brk_ml_count": 2
},
"dimensions": {}
},
{
"metrics": {
"brk_lb_bandwidth_in_usage": 0.0,
"brk_lb_bandwidth_out_usage": 0.0,
"brk_lb_cpu_usage": 13.968855,
"brk_lb_directMemory_usage": 3.125,
"brk_lb_memory_usage": 36.504265
},
"dimensions": {
"broker": "pulsar-devenv-broker-1.pulsar-devenv-broker.pulsar-devenv.svc.cluster.local",
"metric": "loadBalancing"
}
},
{
"metrics": {
"brk_default_pool_allocated": 8388608,
"brk_default_pool_used": 1138688,
"jvm_direct_memory_used": 8388615,
"jvm_gc_old_count": 0,
"jvm_gc_old_pause": 0,
"jvm_gc_young_count": 1,
"jvm_gc_young_pause": 24,
"jvm_heap_used": 98862216,
"jvm_max_direct_memory": 268435456,
"jvm_max_memory": 268435456,
"jvm_thread_cnt": 92,
"jvm_total_memory": 147849216
},
"dimensions": { "metric": "jvm_metrics" }
},
{
"metrics": {
"brk_zk_read_rate_s": 0.0,
"brk_zk_read_time_75percentile_ms": 0,
"brk_zk_read_time_95percentile_ms": 0,
"brk_zk_read_time_99_99_percentile_ms": 0,
"brk_zk_read_time_99_9_percentile_ms": 0,
"brk_zk_read_time_99_percentile_ms": 0,
"brk_zk_read_time_mean_ms": 0,
"brk_zk_read_time_median_ms": 0
},
"dimensions": {
"broker": "pulsar-devenv-broker-1.pulsar-devenv-broker.pulsar-devenv.svc.cluster.local",
"cluster": "pulsar-devenv",
"metric": "zk_read_latency"
}
},
{
"metrics": {
"brk_AddEntryLatencyBuckets_0.0_0.5": 0,
"brk_AddEntryLatencyBuckets_0.5_1.0": 0,
"brk_AddEntryLatencyBuckets_1.0_5.0": 41,
"brk_AddEntryLatencyBuckets_10.0_20.0": 2,
"brk_AddEntryLatencyBuckets_100.0_200.0": 0,
"brk_AddEntryLatencyBuckets_20.0_50.0": 0,
"brk_AddEntryLatencyBuckets_200.0_1000.0": 0,
"brk_AddEntryLatencyBuckets_5.0_10.0": 30,
"brk_AddEntryLatencyBuckets_50.0_100.0": 0,
"brk_AddEntryLatencyBuckets_OVERFLOW": 0,
"brk_in_rate": 198.9759467432553,
"brk_in_tp_rate": 220408.67530764476,
"brk_max_replication_delay_second": 0.0,
"brk_msg_backlog": 0.0,
"brk_no_of_consumers": 0,
"brk_no_of_producers": 2,
"brk_no_of_replicators": 0,
"brk_no_of_subscriptions": 0,
"brk_out_rate": 0.0,
"brk_out_tp_rate": 0.0,
"brk_replication_backlog": 0.0,
"brk_storage_size": 0.0
},
"dimensions": { "namespace": "tenant-1/ns-3" }
},
{
"metrics": {
"brk_zk_write_rate_s": 0.0,
"brk_zk_write_time_75percentile_ms": 0,
"brk_zk_write_time_95percentile_ms": 0,
"brk_zk_write_time_99_99_percentile_ms": 0,
"brk_zk_write_time_99_9_percentile_ms": 0,
"brk_zk_write_time_99_percentile_ms": 0,
"brk_zk_write_time_mean_ms": 0,
"brk_zk_write_time_median_ms": 0
},
"dimensions": {
"broker": "pulsar-devenv-broker-1.pulsar-devenv-broker.pulsar-devenv.svc.cluster.local",
"cluster": "pulsar-devenv",
"metric": "zk_write_latency"
}
},
{
"metrics": {
"brk_active_connections": 8,
"brk_connection_closed_total_count": 1,
"brk_connection_create_fail_count": 0,
"brk_connection_create_success_count": 9,
"brk_connection_created_total_count": 9
},
"dimensions": {
"broker": "pulsar-devenv-broker-1.pulsar-devenv-broker.pulsar-devenv.svc.cluster.local",
"cluster": "pulsar-devenv",
"metric": "broker_connection"
}
},
{
"metrics": {
"brk_topic_load_rate_s": 0.0,
"brk_topic_load_time_75percentile_ms": 0,
"brk_topic_load_time_95percentile_ms": 0,
"brk_topic_load_time_99_99_percentile_ms": 0,
"brk_topic_load_time_99_9_percentile_ms": 0,
"brk_topic_load_time_99_percentile_ms": 0,
"brk_topic_load_time_mean_ms": 0,
"brk_topic_load_time_median_ms": 0
},
"dimensions": {
"broker": "pulsar-devenv-broker-1.pulsar-devenv-broker.pulsar-devenv.svc.cluster.local",
"cluster": "pulsar-devenv",
"metric": "topic_load_times"
}
},
{
"metrics": {
"brk_ml_AddEntryBytesRate": 82377.38333333333,
"brk_ml_AddEntryErrors": 0.0,
"brk_ml_AddEntryLatencyBuckets_0.0_0.5": 0.0,
"brk_ml_AddEntryLatencyBuckets_0.5_1.0": 0.0,
"brk_ml_AddEntryLatencyBuckets_1.0_5.0": 43.41666666666667,
"brk_ml_AddEntryLatencyBuckets_10.0_20.0": 2.05,
"brk_ml_AddEntryLatencyBuckets_100.0_200.0": 0.08333333333333333,
"brk_ml_AddEntryLatencyBuckets_20.0_50.0": 0.6166666666666667,
"brk_ml_AddEntryLatencyBuckets_200.0_1000.0": 0.08333333333333333,
"brk_ml_AddEntryLatencyBuckets_5.0_10.0": 27.766666666666666,
"brk_ml_AddEntryLatencyBuckets_50.0_100.0": 0.23333333333333334,
"brk_ml_AddEntryLatencyBuckets_OVERFLOW": 0.0,
"brk_ml_AddEntryMessagesRate": 74.28333333333333,
"brk_ml_AddEntrySucceed": 4457.0,
"brk_ml_AddEntryWithReplicasBytesRate": 164754.76666666666,
"brk_ml_EntrySizeBuckets_0.0_128.0": 0.0,
"brk_ml_EntrySizeBuckets_1024.0_2048.0": 74.23333333333333,
"brk_ml_EntrySizeBuckets_102400.0_1048576.0": 0.0,
"brk_ml_EntrySizeBuckets_128.0_512.0": 0.0,
"brk_ml_EntrySizeBuckets_16384.0_102400.0": 0.0,
"brk_ml_EntrySizeBuckets_2048.0_4096.0": 0.05,
"brk_ml_EntrySizeBuckets_4096.0_16384.0": 0.0,
"brk_ml_EntrySizeBuckets_512.0_1024.0": 0.0,
"brk_ml_EntrySizeBuckets_OVERFLOW": 0.0,
"brk_ml_LedgerAddEntryLatencyBuckets_0.0_0.5": 0.0,
"brk_ml_LedgerAddEntryLatencyBuckets_0.5_1.0": 0.0,
"brk_ml_LedgerAddEntryLatencyBuckets_1.0_5.0": 43.61666666666667,
"brk_ml_LedgerAddEntryLatencyBuckets_10.0_20.0": 2.05,
"brk_ml_LedgerAddEntryLatencyBuckets_100.0_200.0": 0.08333333333333333,
"brk_ml_LedgerAddEntryLatencyBuckets_20.0_50.0": 0.6166666666666667,
"brk_ml_LedgerAddEntryLatencyBuckets_200.0_1000.0": 0.08333333333333333,
"brk_ml_LedgerAddEntryLatencyBuckets_5.0_10.0": 27.566666666666663,
"brk_ml_LedgerAddEntryLatencyBuckets_50.0_100.0": 0.23333333333333334,
"brk_ml_LedgerAddEntryLatencyBuckets_OVERFLOW": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_0.0_0.5": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_0.5_1.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_1.0_5.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_10.0_20.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_100.0_200.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_20.0_50.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_200.0_1000.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_5.0_10.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_50.0_100.0": 0.0,
"brk_ml_LedgerSwitchLatencyBuckets_OVERFLOW": 0.0,
"brk_ml_MarkDeleteRate": 0.0,
"brk_ml_NumberOfMessagesInBacklog": 0.0,
"brk_ml_ReadEntriesBytesRate": 0.0,
"brk_ml_ReadEntriesErrors": 0.0,
"brk_ml_ReadEntriesRate": 0.0,
"brk_ml_ReadEntriesSucceeded": 0.0,
"brk_ml_StoredMessagesSize": 3.377775e7
},
"dimensions": { "namespace": "tenant-1/ns-3/persistent" }
}
] |
|
There is defintely some activity in Screen.Recording.2022-09-15.at.11.43.32.AM.mov |
|
@visortelle - how many brokers are you running? The broker stats command only gets stats from the broker that serves the http request. If you want the stats for the whole cluster, you currently have to make the broker stats command to all brokers in the cluster. This nuance is important because if a namespace does not have any bundles on a broker, you will not get the stats for that namespace when getting that broker's stats. |
|
@michaeljmarshall thank you for the reply. I suspected that this is how it works. I just naively hoped that Pulsar Proxy or something else can automagically 🪄 handle it for me. Two more related questions:
|
@visortelle - how many brokers are you running? The broker stats command only gets stats from the broker that serves the http request. If you want the stats for the whole cluster, you currently have to make the broker stats command to all brokers in the cluster. This nuance is important because if a namespace does not have any bundles on a broker, you will not get the stats for that namespace when getting that broker's stats.