From dd238358934f47286a9371c97f653127778b2586 Mon Sep 17 00:00:00 2001 From: daliu Date: Fri, 2 Feb 2024 14:54:09 +0800 Subject: [PATCH 01/11] Add io-metrics to topic and broker Signed-off-by: daliu --- .../kafka/ui/controller/TopicsController.java | 12 +++ .../kafka/ui/model/InternalTopic.java | 63 +++++++++++- .../com/provectus/kafka/ui/model/Metrics.java | 24 +++++ .../ui/service/metrics/WellKnownMetrics.java | 98 ++++++++++++++++--- .../main/resources/swagger/kafka-ui-api.yaml | 30 ++++++ .../Brokers/BrokersList/BrokersList.tsx | 4 + .../src/components/Topics/List/TopicTable.tsx | 48 +++++++++ 7 files changed, 265 insertions(+), 14 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/TopicsController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/TopicsController.java index d24044717f3..59693b2e38e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/TopicsController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/TopicsController.java @@ -371,6 +371,18 @@ private Comparator getComparatorForTopic( return Comparator.comparing(InternalTopic::getReplicationFactor); case SIZE: return Comparator.comparing(InternalTopic::getSegmentSize); + case BYTESIN_PERSEC: + return Comparator.comparing(InternalTopic::getBytesInPerSec); + case BYTESOUT_PERSEC: + return Comparator.comparing(InternalTopic::getBytesOutPerSec); + case MSG_RATE: + return Comparator.comparing(InternalTopic::getMessageInMeanRate); + case MSG_5_RATE: + return Comparator.comparing(InternalTopic::getMessageInFiveMinuteRate); + case FETCH_RATE: + return Comparator.comparing(InternalTopic::getFetchRequestsMeanRate); + case FETCH_5_RATE: + return Comparator.comparing(InternalTopic::getFetchRequestsFiveMinuteRate); case NAME: default: return defaultComparator; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java index 43a6012d215..c0392f68217 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java @@ -2,6 +2,7 @@ import com.provectus.kafka.ui.config.ClustersProperties; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -35,6 +36,18 @@ public class InternalTopic { // rates from metrics private final BigDecimal bytesInPerSec; private final BigDecimal bytesOutPerSec; + private final BigDecimal messageInMeanRate; + private final BigDecimal messageInOneMinuteRate; + private final BigDecimal messageInFiveMinuteRate; + private final BigDecimal messageInFifteenMinuteRate; + private final BigDecimal fetchRequestsMeanRate; + private final BigDecimal fetchRequestsOneMinuteRate; + private final BigDecimal fetchRequestsFiveMinuteRate; + private final BigDecimal fetchRequestsFifteenMinuteRate; + private final BigDecimal produceRequestsMeanRate; + private final BigDecimal produceRequestsOneMinuteRate; + private final BigDecimal produceRequestsFiveMinuteRate; + private final BigDecimal produceRequestsFifteenMinuteRate; // from log dir data private final long segmentSize; @@ -114,8 +127,54 @@ public static InternalTopic from(TopicDescription topicDescription, topic.segmentSize(segmentStats.getSegmentSize()); } - topic.bytesInPerSec(metrics.getTopicBytesInPerSec().get(topicDescription.name())); - topic.bytesOutPerSec(metrics.getTopicBytesOutPerSec().get(topicDescription.name())); + topic.bytesInPerSec(metrics.getTopicBytesInPerSec().get(topicDescription.name()) == null + ? BigDecimal.ZERO : metrics.getTopicBytesInPerSec().get(topicDescription.name())); + topic.bytesOutPerSec(metrics.getTopicBytesOutPerSec().get(topicDescription.name()) == null + ? BigDecimal.ZERO : metrics.getTopicBytesOutPerSec().get(topicDescription.name())); + + topic.messageInMeanRate(metrics.getMessageInMeanRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getMessageInMeanRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.messageInOneMinuteRate(metrics.getMessageInOneMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getMessageInOneMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.messageInFiveMinuteRate(metrics.getMessageInFiveMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getMessageInFiveMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.messageInFifteenMinuteRate(metrics.getMessageInFifteenMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getMessageInFifteenMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + + topic.fetchRequestsMeanRate(metrics.getFetchRequestsMeanRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getFetchRequestsMeanRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.fetchRequestsOneMinuteRate(metrics.getFetchRequestsOneMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getFetchRequestsOneMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.fetchRequestsFiveMinuteRate(metrics.getFetchRequestsFiveMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getFetchRequestsFiveMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.fetchRequestsFifteenMinuteRate(metrics.getFetchRequestsFifteenMinuteRate().get( + topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getFetchRequestsFifteenMinuteRate().get(topicDescription.name()) + .setScale(2, RoundingMode.HALF_UP)); + + topic.produceRequestsMeanRate(metrics.getProduceRequestsMeanRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getProduceRequestsMeanRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.produceRequestsOneMinuteRate(metrics.getProduceRequestsOneMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getProduceRequestsOneMinuteRate().get(topicDescription.name()).setScale(2, RoundingMode.HALF_UP)); + topic.produceRequestsFiveMinuteRate(metrics.getProduceRequestsFiveMinuteRate().get(topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getProduceRequestsFiveMinuteRate().get(topicDescription.name()) + .setScale(2, RoundingMode.HALF_UP)); + topic.produceRequestsFifteenMinuteRate(metrics.getProduceRequestsFifteenMinuteRate().get( + topicDescription.name()) == null + ? BigDecimal.ZERO + : metrics.getProduceRequestsFifteenMinuteRate().get(topicDescription.name()) + .setScale(2, RoundingMode.HALF_UP)); topic.topicConfigs( configs.stream().map(InternalTopicConfig::from).collect(Collectors.toList())); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java index 02bfe6dea13..d65d5e91508 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java @@ -21,6 +21,18 @@ public class Metrics { Map topicBytesInPerSec; Map topicBytesOutPerSec; Map> perBrokerMetrics; + Map messageInMeanRate; + Map messageInOneMinuteRate; + Map messageInFiveMinuteRate; + Map messageInFifteenMinuteRate; + Map fetchRequestsMeanRate; + Map fetchRequestsOneMinuteRate; + Map fetchRequestsFiveMinuteRate; + Map fetchRequestsFifteenMinuteRate; + Map produceRequestsMeanRate; + Map produceRequestsOneMinuteRate; + Map produceRequestsFiveMinuteRate; + Map produceRequestsFifteenMinuteRate; public static Metrics empty() { return Metrics.builder() @@ -29,6 +41,18 @@ public static Metrics empty() { .topicBytesInPerSec(Map.of()) .topicBytesOutPerSec(Map.of()) .perBrokerMetrics(Map.of()) + .messageInMeanRate(Map.of()) + .messageInOneMinuteRate(Map.of()) + .messageInFiveMinuteRate(Map.of()) + .messageInFifteenMinuteRate(Map.of()) + .fetchRequestsMeanRate(Map.of()) + .fetchRequestsOneMinuteRate(Map.of()) + .fetchRequestsFiveMinuteRate(Map.of()) + .fetchRequestsFifteenMinuteRate(Map.of()) + .produceRequestsMeanRate(Map.of()) + .produceRequestsOneMinuteRate(Map.of()) + .produceRequestsFiveMinuteRate(Map.of()) + .produceRequestsFifteenMinuteRate(Map.of()) .build(); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java index 8dd4609b601..fda23365890 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java @@ -12,6 +12,9 @@ class WellKnownMetrics { private static final String BROKER_TOPIC_METRICS = "BrokerTopicMetrics"; + private static final String MEAN_RATE = "MeanRate"; + private static final String ONE_MINUTE_RATE = "OneMinuteRate"; + private static final String FIVE_MINUTE_RATE = "FiveMinuteRate"; private static final String FIFTEEN_MINUTE_RATE = "FifteenMinuteRate"; // per broker @@ -21,6 +24,18 @@ class WellKnownMetrics { // per topic final Map bytesInFifteenMinuteRate = new HashMap<>(); final Map bytesOutFifteenMinuteRate = new HashMap<>(); + final Map messageInMeanRate = new HashMap<>(); + final Map messageInOneMinuteRate = new HashMap<>(); + final Map messageInFiveMinuteRate = new HashMap<>(); + final Map messageInFifteenMinuteRate = new HashMap<>(); + final Map fetchRequestsMeanRate = new HashMap<>(); + final Map fetchRequestsOneMinuteRate = new HashMap<>(); + final Map fetchRequestsFiveMinuteRate = new HashMap<>(); + final Map fetchRequestsFifteenMinuteRate = new HashMap<>(); + final Map produceRequestsMeanRate = new HashMap<>(); + final Map produceRequestsOneMinuteRate = new HashMap<>(); + final Map produceRequestsFiveMinuteRate = new HashMap<>(); + final Map produceRequestsFifteenMinuteRate = new HashMap<>(); void populate(Node node, RawMetric rawMetric) { updateBrokerIOrates(node, rawMetric); @@ -32,6 +47,18 @@ void apply(Metrics.MetricsBuilder metricsBuilder) { metricsBuilder.topicBytesOutPerSec(bytesOutFifteenMinuteRate); metricsBuilder.brokerBytesInPerSec(brokerBytesInFifteenMinuteRate); metricsBuilder.brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate); + metricsBuilder.messageInMeanRate(messageInMeanRate); + metricsBuilder.messageInOneMinuteRate(messageInOneMinuteRate); + metricsBuilder.messageInFiveMinuteRate(messageInFiveMinuteRate); + metricsBuilder.messageInFifteenMinuteRate(messageInFifteenMinuteRate); + metricsBuilder.fetchRequestsMeanRate(fetchRequestsMeanRate); + metricsBuilder.fetchRequestsOneMinuteRate(fetchRequestsOneMinuteRate); + metricsBuilder.fetchRequestsFiveMinuteRate(fetchRequestsFiveMinuteRate); + metricsBuilder.fetchRequestsFifteenMinuteRate(fetchRequestsFifteenMinuteRate); + metricsBuilder.produceRequestsMeanRate(produceRequestsMeanRate); + metricsBuilder.produceRequestsOneMinuteRate(produceRequestsOneMinuteRate); + metricsBuilder.produceRequestsFiveMinuteRate(produceRequestsFiveMinuteRate); + metricsBuilder.produceRequestsFifteenMinuteRate(produceRequestsFifteenMinuteRate); } private void updateBrokerIOrates(Node node, RawMetric rawMetric) { @@ -53,18 +80,65 @@ && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { } private void updateTopicsIOrates(RawMetric rawMetric) { - String name = rawMetric.name(); - String topic = rawMetric.labels().get("topic"); - if (topic != null - && containsIgnoreCase(name, BROKER_TOPIC_METRICS) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { - String nameProperty = rawMetric.labels().get("name"); - if ("BytesInPerSec".equalsIgnoreCase(nameProperty)) { - bytesInFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } else if ("BytesOutPerSec".equalsIgnoreCase(nameProperty)) { - bytesOutFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } - } + String name = rawMetric.name(); + String topic = rawMetric.labels().get("topic"); + if (topic != null && containsIgnoreCase(name, BROKER_TOPIC_METRICS)) { + if (endsWithIgnoreCase(name, MEAN_RATE)) { + String nameProperty = rawMetric.labels().get("name"); + if ("MessagesInPerSec".equalsIgnoreCase(nameProperty)) { + messageInMeanRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalFetchRequestsPerSec".equalsIgnoreCase(nameProperty)) { + fetchRequestsMeanRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalProduceRequestsPerSec".equalsIgnoreCase(nameProperty)) { + produceRequestsMeanRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } + }else if (endsWithIgnoreCase(name, ONE_MINUTE_RATE)) { + String nameProperty = rawMetric.labels().get("name"); + if ("MessagesInPerSec".equalsIgnoreCase(nameProperty)) { + messageInOneMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalFetchRequestsPerSec".equalsIgnoreCase(nameProperty)) { + fetchRequestsOneMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalProduceRequestsPerSec".equalsIgnoreCase(nameProperty)) { + produceRequestsOneMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } + }else if (endsWithIgnoreCase(name, FIVE_MINUTE_RATE)) { + String nameProperty = rawMetric.labels().get("name"); + if ("MessagesInPerSec".equalsIgnoreCase(nameProperty)) { + messageInFiveMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalFetchRequestsPerSec".equalsIgnoreCase(nameProperty)) { + fetchRequestsFiveMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalProduceRequestsPerSec".equalsIgnoreCase(nameProperty)) { + produceRequestsFiveMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } + }else if (endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { + String nameProperty = rawMetric.labels().get("name"); + if ("BytesInPerSec".equalsIgnoreCase(nameProperty)) { + bytesInFifteenMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("BytesOutPerSec".equalsIgnoreCase(nameProperty)) { + bytesOutFifteenMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("MessagesInPerSec".equalsIgnoreCase(nameProperty)) { + messageInFifteenMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalFetchRequestsPerSec".equalsIgnoreCase(nameProperty)) { + fetchRequestsFifteenMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } else if ("TotalProduceRequestsPerSec".equalsIgnoreCase(nameProperty)) { + produceRequestsFifteenMinuteRate.compute(topic, + (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); + } + } + } } } diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index ae51d31568f..57408102401 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -2318,6 +2318,12 @@ components: - TOTAL_PARTITIONS - REPLICATION_FACTOR - SIZE + - BYTESIN_PERSEC + - BYTESOUT_PERSEC + - MSG_RATE + - MSG_5_RATE + - FETCH_RATE + - FETCH_5_RATE ConnectorColumnsToSort: type: string @@ -2357,6 +2363,30 @@ components: type: number bytesOutPerSec: type: number + messageInMeanRate: + type: number + messageInOneMinuteRate: + type: number + messageInFiveMinuteRate: + type: number + messageInFifteenMinuteRate: + type: number + fetchRequestsMeanRate: + type: number + fetchRequestsOneMinuteRate: + type: number + fetchRequestsFiveMinuteRate: + type: number + fetchRequestsFifteenMinuteRate: + type: number + produceRequestsMeanRate: + type: number + produceRequestsOneMinuteRate: + type: number + produceRequestsFiveMinuteRate: + type: number + produceRequestsFifteenMinuteRate: + type: number underReplicatedPartitions: type: integer cleanUpPolicy: diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx index ede570c655b..99e065e9436 100644 --- a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx @@ -63,6 +63,8 @@ const BrokersList: React.FC = () => { partitionsSkew: broker?.partitionsSkew, leadersSkew: broker?.leadersSkew, inSyncPartitions: broker?.inSyncPartitions, + bytesInPerSec: broker?.bytesInPerSec, + bytesOutPerSec: broker?.bytesOutPerSec, }; }); }, [diskUsage, brokers]); @@ -160,6 +162,8 @@ const BrokersList: React.FC = () => { header: 'Host', accessorKey: 'host', }, + { header: 'IN /sec', accessorKey: 'bytesInPerSec', cell: SizeCell}, + { header: 'OUT /sec', accessorKey: 'bytesOutPerSec', cell: SizeCell}, ], [] ); diff --git a/kafka-ui-react-app/src/components/Topics/List/TopicTable.tsx b/kafka-ui-react-app/src/components/Topics/List/TopicTable.tsx index dcafa61b915..e57cc9c5fcd 100644 --- a/kafka-ui-react-app/src/components/Topics/List/TopicTable.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/TopicTable.tsx @@ -65,6 +65,54 @@ const TopicTable: React.FC = () => { accessorKey: 'replicationFactor', enableSorting: false, }, + { + id: TopicColumnsToSort.BYTESIN_PERSEC, + header: 'BytesIn /sec', + accessorKey: 'bytesInPerSec', + enableSorting: true, + cell: SizeCell, + }, + { + id: TopicColumnsToSort.BYTESOUT_PERSEC, + header: 'BytesOut /sec', + accessorKey: 'bytesOutPerSec', + enableSorting: true, + cell: SizeCell, + }, + { + id: TopicColumnsToSort.MSG_RATE, + header: 'Msg /sec', + accessorKey: 'messageInMeanRate', + enableSorting: true, + }, + { + id: TopicColumnsToSort.MSG_5_RATE, + header: 'Msg-5min /sec', + accessorKey: 'messageInFiveMinuteRate', + enableSorting: true, + }, + { + id: TopicColumnsToSort.FETCH_RATE, + header: 'Fetch /sec', + accessorKey: 'fetchRequestsMeanRate', + enableSorting: true, + }, + { + id: TopicColumnsToSort.FETCH_5_RATE, + header: 'Fetch-5min /sec', + accessorKey: 'fetchRequestsFiveMinuteRate', + enableSorting: true, + }, + { + header: 'Produce /sec', + accessorKey: 'produceRequestsMeanRate', + enableSorting: false, + }, + { + header: 'Produce-5min /sec', + accessorKey: 'produceRequestsFiveMinuteRate', + enableSorting: false, + }, { header: 'Number of messages', accessorKey: 'partitions', From 3f4561329fab38adca34333e33d74a361371730f Mon Sep 17 00:00:00 2001 From: daliu Date: Fri, 2 Feb 2024 15:07:35 +0800 Subject: [PATCH 02/11] Format Signed-off-by: daliu --- .../src/components/Brokers/BrokersList/BrokersList.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx index 99e065e9436..f51e579b5a9 100644 --- a/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx +++ b/kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx @@ -162,8 +162,8 @@ const BrokersList: React.FC = () => { header: 'Host', accessorKey: 'host', }, - { header: 'IN /sec', accessorKey: 'bytesInPerSec', cell: SizeCell}, - { header: 'OUT /sec', accessorKey: 'bytesOutPerSec', cell: SizeCell}, + { header: 'IN /sec', accessorKey: 'bytesInPerSec', cell: SizeCell }, + { header: 'OUT /sec', accessorKey: 'bytesOutPerSec', cell: SizeCell }, ], [] ); From cd233e3f8961d1583d00588e73d063fd9d1e001b Mon Sep 17 00:00:00 2001 From: daliu Date: Fri, 2 Feb 2024 15:58:29 +0800 Subject: [PATCH 03/11] Test case Signed-off-by: daliu --- .../src/components/Topics/List/__tests__/TopicTable.spec.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index 61be3f1360c..a39c7dbf9a4 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -98,13 +98,13 @@ describe('TopicTable Components', () => { screen.getByRole('link', { name: '__internal.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: '__internal.topic 1 0 1 0 0 Bytes' }) + screen.getByRole('row', { name: '__internal.topic 1 0 1 0 Bytes Bytes 0 0 0 0 0 0 0 Bytes' }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'external.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: 'external.topic 1 0 1 0 1 KB' }) + screen.getByRole('row', { name: 'external.topic 1 0 1 0 KB KB 0 0 0 0 0 0 0 1 KB' }) ).toBeInTheDocument(); expect(screen.getAllByRole('checkbox').length).toEqual(3); From 45f1471900f2cb8902e37c1bb1ed8e7d0b2e3fb1 Mon Sep 17 00:00:00 2001 From: daliu Date: Fri, 2 Feb 2024 16:01:31 +0800 Subject: [PATCH 04/11] Test case Signed-off-by: daliu --- .../components/Topics/List/__tests__/TopicTable.spec.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index a39c7dbf9a4..47ee9f94287 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -98,13 +98,17 @@ describe('TopicTable Components', () => { screen.getByRole('link', { name: '__internal.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: '__internal.topic 1 0 1 0 Bytes Bytes 0 0 0 0 0 0 0 Bytes' }) + screen.getByRole('row', { + name: '__internal.topic 1 0 1 0 Bytes Bytes 0 0 0 0 0 0 0 Bytes', + }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'external.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: 'external.topic 1 0 1 0 KB KB 0 0 0 0 0 0 0 1 KB' }) + screen.getByRole('row', { + name: 'external.topic 1 0 1 0 KB KB 0 0 0 0 0 0 0 1 KB', + }) ).toBeInTheDocument(); expect(screen.getAllByRole('checkbox').length).toEqual(3); From 77d4b5f48fd0348278f105e4fbc92bdb3f3ea1fb Mon Sep 17 00:00:00 2001 From: daliu Date: Fri, 2 Feb 2024 16:05:06 +0800 Subject: [PATCH 05/11] Recover Signed-off-by: daliu --- .../components/Topics/List/__tests__/TopicTable.spec.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index 47ee9f94287..61be3f1360c 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -98,17 +98,13 @@ describe('TopicTable Components', () => { screen.getByRole('link', { name: '__internal.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { - name: '__internal.topic 1 0 1 0 Bytes Bytes 0 0 0 0 0 0 0 Bytes', - }) + screen.getByRole('row', { name: '__internal.topic 1 0 1 0 0 Bytes' }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'external.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { - name: 'external.topic 1 0 1 0 KB KB 0 0 0 0 0 0 0 1 KB', - }) + screen.getByRole('row', { name: 'external.topic 1 0 1 0 1 KB' }) ).toBeInTheDocument(); expect(screen.getAllByRole('checkbox').length).toEqual(3); From b6818f6542a17dfe4149b5ede77e2ea37cac0e9d Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 16:39:35 +0800 Subject: [PATCH 06/11] BugFix ActiveController not correct Signed-off-by: daliu --- .../kafka/ui/model/InternalClusterState.java | 9 ++++++--- .../com/provectus/kafka/ui/model/Metrics.java | 1 + .../service/metrics/JmxMetricsRetriever.java | 5 +++++ .../ui/service/metrics/WellKnownMetrics.java | 18 ++++++++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java index 28e9a7413a3..77f2fc14a8e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java @@ -38,9 +38,12 @@ public InternalClusterState(KafkaCluster cluster, Statistics statistics) { .orElse(null); topicCount = statistics.getTopicDescriptions().size(); brokerCount = statistics.getClusterDescription().getNodes().size(); - activeControllers = Optional.ofNullable(statistics.getClusterDescription().getController()) - .map(Node::id) - .orElse(null); + activeControllers = statistics.getMetrics().getController(); + if (activeControllers == null || activeControllers < 0) { + activeControllers = Optional.ofNullable(statistics.getClusterDescription().getController()) + .map(Node::id) + .orElse(null); + } version = statistics.getVersion(); if (statistics.getLogDirInfo() != null) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java index d65d5e91508..b312ded52c8 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java @@ -33,6 +33,7 @@ public class Metrics { Map produceRequestsOneMinuteRate; Map produceRequestsFiveMinuteRate; Map produceRequestsFifteenMinuteRate; + Integer controller; public static Metrics empty() { return Metrics.builder() diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java index e7a58cbae27..f25be7e55f3 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java @@ -37,6 +37,7 @@ class JmxMetricsRetriever implements MetricsRetriever, Closeable { private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; private static final String CANONICAL_NAME_PATTERN = "kafka.server*:*"; + private static final String CONTROLLER_NAME_PATTERN = "kafka.controller*:*"; @Override public void close() { @@ -118,6 +119,10 @@ private void getMetricsFromJmx(JMXConnector jmxConnector, List sink) for (ObjectName jmxMetric : jmxMetrics) { sink.addAll(extractObjectMetrics(jmxMetric, msc)); } + var controllerMetrics = msc.queryNames(new ObjectName(CONTROLLER_NAME_PATTERN), null); + for (ObjectName jmxMetric : controllerMetrics) { + sink.addAll(extractObjectMetrics(jmxMetric, msc)); + } } @SneakyThrows diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java index fda23365890..e37c3ea7976 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java @@ -36,9 +36,11 @@ class WellKnownMetrics { final Map produceRequestsOneMinuteRate = new HashMap<>(); final Map produceRequestsFiveMinuteRate = new HashMap<>(); final Map produceRequestsFifteenMinuteRate = new HashMap<>(); - + int controller = -1; + void populate(Node node, RawMetric rawMetric) { updateBrokerIOrates(node, rawMetric); + updateController(node, rawMetric); updateTopicsIOrates(rawMetric); } @@ -59,6 +61,7 @@ void apply(Metrics.MetricsBuilder metricsBuilder) { metricsBuilder.produceRequestsOneMinuteRate(produceRequestsOneMinuteRate); metricsBuilder.produceRequestsFiveMinuteRate(produceRequestsFiveMinuteRate); metricsBuilder.produceRequestsFifteenMinuteRate(produceRequestsFifteenMinuteRate); + metricsBuilder.controller(controller); } private void updateBrokerIOrates(Node node, RawMetric rawMetric) { @@ -140,5 +143,16 @@ private void updateTopicsIOrates(RawMetric rawMetric) { } } } - + + private void updateController(Node node, RawMetric rawMetric) { + if (controller < 0 && containsIgnoreCase(rawMetric.name(), "KafkaController")) { + String nameProperty = rawMetric.labels().get("name"); + if ("ActiveControllerCount".equals(nameProperty)) { + BigDecimal value = rawMetric.value(); + if (value.intValue() == 1) { + controller = node.id(); + } + } + } + } } From 1b37162d16ffac42135f2bf11f0d0b24d3f7a531 Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 16:51:40 +0800 Subject: [PATCH 07/11] For pass test cases Signed-off-by: daliu --- .../src/components/Topics/List/__tests__/TopicTable.spec.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index 61be3f1360c..b5eb39b5f81 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -98,13 +98,13 @@ describe('TopicTable Components', () => { screen.getByRole('link', { name: '__internal.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: '__internal.topic 1 0 1 0 0 Bytes' }) + screen.getByRole('row', { name: '__internal.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 0 Bytes' }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'external.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: 'external.topic 1 0 1 0 1 KB' }) + screen.getByRole('row', { name: 'external.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 1 KB' }) ).toBeInTheDocument(); expect(screen.getAllByRole('checkbox').length).toEqual(3); From 220a3064448871f6b86fa10a212d85bf702fc9e1 Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 16:54:24 +0800 Subject: [PATCH 08/11] fix eslint problem Signed-off-by: daliu --- .../components/Topics/List/__tests__/TopicTable.spec.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index b5eb39b5f81..752958cfd25 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -98,13 +98,17 @@ describe('TopicTable Components', () => { screen.getByRole('link', { name: '__internal.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: '__internal.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 0 Bytes' }) + screen.getByRole('row', { + name: '__internal.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 0 Bytes', + }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'external.topic' }) ).toBeInTheDocument(); expect( - screen.getByRole('row', { name: 'external.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 1 KB' }) + screen.getByRole('row', { + name: 'external.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 1 KB', + }) ).toBeInTheDocument(); expect(screen.getAllByRole('checkbox').length).toEqual(3); From ac0c1d2977e53920f0d6a9a6f073942c84c55aaf Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 17:01:04 +0800 Subject: [PATCH 09/11] Adapt test case Signed-off-by: daliu --- .../src/components/Topics/List/__tests__/TopicTable.spec.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx index 752958cfd25..4d067787446 100644 --- a/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/kafka-ui-react-app/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -221,7 +221,7 @@ describe('TopicTable Components', () => { ).toEqual(2); // Internal topic action buttons are disabled const internalTopicRow = screen.getByRole('row', { - name: '__internal.topic 1 0 1 0 0 Bytes', + name: '__internal.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 0 Bytes', }); expect(internalTopicRow).toBeInTheDocument(); expect( @@ -231,7 +231,7 @@ describe('TopicTable Components', () => { ).toBeDisabled(); // External topic action buttons are enabled const externalTopicRow = screen.getByRole('row', { - name: 'external.topic 1 0 1 0 1 KB', + name: 'external.topic 1 0 1 0 Bytes 0 Bytes 0 0 0 0 0 0 0 1 KB', }); expect(externalTopicRow).toBeInTheDocument(); const extBtn = within(externalTopicRow).getByRole('button', { From 6a4527334a6086aed0b606a8e0d510155d6f260a Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 17:10:51 +0800 Subject: [PATCH 10/11] Test data Signed-off-by: daliu --- kafka-ui-react-app/src/lib/fixtures/topics.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kafka-ui-react-app/src/lib/fixtures/topics.ts b/kafka-ui-react-app/src/lib/fixtures/topics.ts index cff119077a8..d0c582fe126 100644 --- a/kafka-ui-react-app/src/lib/fixtures/topics.ts +++ b/kafka-ui-react-app/src/lib/fixtures/topics.ts @@ -12,6 +12,14 @@ export const internalTopicPayload = { internal: true, partitionCount: 1, replicationFactor: 1, + bytesInPerSec: 0, + bytesOutPerSec: 0, + messageInMeanRate: 0, + messageInFiveMinuteRate: 0, + fetchRequestsMeanRate:0, + fetchRequestsFiveMinuteRate: 0, + produceRequestsMeanRate: 0, + produceRequestsFiveMinuteRate: 0, replicas: 1, inSyncReplicas: 1, segmentSize: 0, @@ -33,6 +41,14 @@ export const externalTopicPayload = { internal: false, partitionCount: 1, replicationFactor: 1, + bytesInPerSec: 0, + bytesOutPerSec: 0, + messageInMeanRate: 0, + messageInFiveMinuteRate: 0, + fetchRequestsMeanRate:0, + fetchRequestsFiveMinuteRate: 0, + produceRequestsMeanRate: 0, + produceRequestsFiveMinuteRate: 0, replicas: 1, inSyncReplicas: 1, segmentSize: 1263, From 93b8cf0c46ce179ce4b4b091ea408be7f7c5a62f Mon Sep 17 00:00:00 2001 From: daliu Date: Sun, 4 Feb 2024 17:13:42 +0800 Subject: [PATCH 11/11] eslint Signed-off-by: daliu --- kafka-ui-react-app/src/lib/fixtures/topics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-ui-react-app/src/lib/fixtures/topics.ts b/kafka-ui-react-app/src/lib/fixtures/topics.ts index d0c582fe126..b8df46e9693 100644 --- a/kafka-ui-react-app/src/lib/fixtures/topics.ts +++ b/kafka-ui-react-app/src/lib/fixtures/topics.ts @@ -16,7 +16,7 @@ export const internalTopicPayload = { bytesOutPerSec: 0, messageInMeanRate: 0, messageInFiveMinuteRate: 0, - fetchRequestsMeanRate:0, + fetchRequestsMeanRate: 0, fetchRequestsFiveMinuteRate: 0, produceRequestsMeanRate: 0, produceRequestsFiveMinuteRate: 0, @@ -45,7 +45,7 @@ export const externalTopicPayload = { bytesOutPerSec: 0, messageInMeanRate: 0, messageInFiveMinuteRate: 0, - fetchRequestsMeanRate:0, + fetchRequestsMeanRate: 0, fetchRequestsFiveMinuteRate: 0, produceRequestsMeanRate: 0, produceRequestsFiveMinuteRate: 0,