diff --git a/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DO.java b/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DO.java index 3b27d2c6d..fdfcb65ac 100644 --- a/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DO.java +++ b/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DO.java @@ -18,6 +18,8 @@ public class AggTaskV1DO { private Long version; + private String refId; + private Integer deleted; private String json; @@ -62,6 +64,14 @@ public void setVersion(Long version) { this.version = version; } + public String getRefId() { + return refId; + } + + public void setRefId(String refId) { + this.refId = refId == null ? null : refId.trim(); + } + public Integer getDeleted() { return deleted; } @@ -89,6 +99,7 @@ public String toString() { sb.append(", gmtModified=").append(gmtModified); sb.append(", aggId=").append(aggId); sb.append(", version=").append(version); + sb.append(", refId=").append(refId); sb.append(", deleted=").append(deleted); sb.append(", json=").append(json); sb.append("]"); @@ -116,6 +127,8 @@ public boolean equals(Object that) { : this.getAggId().equals(other.getAggId())) && (this.getVersion() == null ? other.getVersion() == null : this.getVersion().equals(other.getVersion())) + && (this.getRefId() == null ? other.getRefId() == null + : this.getRefId().equals(other.getRefId())) && (this.getDeleted() == null ? other.getDeleted() == null : this.getDeleted().equals(other.getDeleted())) && (this.getJson() == null ? other.getJson() == null @@ -131,6 +144,7 @@ public int hashCode() { result = prime * result + ((getGmtModified() == null) ? 0 : getGmtModified().hashCode()); result = prime * result + ((getAggId() == null) ? 0 : getAggId().hashCode()); result = prime * result + ((getVersion() == null) ? 0 : getVersion().hashCode()); + result = prime * result + ((getRefId() == null) ? 0 : getRefId().hashCode()); result = prime * result + ((getDeleted() == null) ? 0 : getDeleted().hashCode()); result = prime * result + ((getJson() == null) ? 0 : getJson().hashCode()); return result; @@ -172,6 +186,11 @@ public Builder version(Long version) { return this; } + public Builder refId(String refId) { + obj.setRefId(refId); + return this; + } + public Builder deleted(Integer deleted) { obj.setDeleted(deleted); return this; @@ -190,8 +209,9 @@ public AggTaskV1DO build() { public enum Column { id("id", "id", "BIGINT", false), gmtCreate("gmt_create", "gmtCreate", "TIMESTAMP", false), gmtModified("gmt_modified", "gmtModified", "TIMESTAMP", false), aggId("agg_id", - "aggId", "VARCHAR", false), version("version", "version", "BIGINT", false), deleted( - "deleted", "deleted", "INTEGER", false), json("json", "json", "LONGVARCHAR", false); + "aggId", "VARCHAR", false), version("version", "version", "BIGINT", + false), refId("ref_id", "refId", "VARCHAR", false), deleted("deleted", "deleted", + "INTEGER", false), json("json", "json", "LONGVARCHAR", false); private static final String BEGINNING_DELIMITER = "`"; diff --git a/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DOExample.java b/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DOExample.java index 23c5aa5bc..04e1b2229 100644 --- a/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DOExample.java +++ b/server/common/common-dao-gen/src/main/java/io/holoinsight/server/common/dao/entity/AggTaskV1DOExample.java @@ -666,6 +666,109 @@ public Criteria andVersionNotBetween(Long value1, Long value2) { return (Criteria) this; } + public Criteria andRefIdIsNull() { + addCriterion("ref_id is null"); + return (Criteria) this; + } + + public Criteria andRefIdIsNotNull() { + addCriterion("ref_id is not null"); + return (Criteria) this; + } + + public Criteria andRefIdEqualTo(String value) { + addCriterion("ref_id =", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdEqualToColumn(AggTaskV1DO.Column column) { + addCriterion(new StringBuilder("ref_id = ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdNotEqualTo(String value) { + addCriterion("ref_id <>", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdNotEqualToColumn(AggTaskV1DO.Column column) { + addCriterion( + new StringBuilder("ref_id <> ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdGreaterThan(String value) { + addCriterion("ref_id >", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdGreaterThanColumn(AggTaskV1DO.Column column) { + addCriterion(new StringBuilder("ref_id > ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdGreaterThanOrEqualTo(String value) { + addCriterion("ref_id >=", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdGreaterThanOrEqualToColumn(AggTaskV1DO.Column column) { + addCriterion( + new StringBuilder("ref_id >= ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdLessThan(String value) { + addCriterion("ref_id <", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdLessThanColumn(AggTaskV1DO.Column column) { + addCriterion(new StringBuilder("ref_id < ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdLessThanOrEqualTo(String value) { + addCriterion("ref_id <=", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdLessThanOrEqualToColumn(AggTaskV1DO.Column column) { + addCriterion( + new StringBuilder("ref_id <= ").append(column.getEscapedColumnName()).toString()); + return (Criteria) this; + } + + public Criteria andRefIdLike(String value) { + addCriterion("ref_id like", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdNotLike(String value) { + addCriterion("ref_id not like", value, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdIn(List values) { + addCriterion("ref_id in", values, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdNotIn(List values) { + addCriterion("ref_id not in", values, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdBetween(String value1, String value2) { + addCriterion("ref_id between", value1, value2, "refId"); + return (Criteria) this; + } + + public Criteria andRefIdNotBetween(String value1, String value2) { + addCriterion("ref_id not between", value1, value2, "refId"); + return (Criteria) this; + } + public Criteria andDeletedIsNull() { addCriterion("deleted is null"); return (Criteria) this; diff --git a/server/common/common-dao-gen/src/main/resources/sqlmap/AggTaskV1DOMapper.xml b/server/common/common-dao-gen/src/main/resources/sqlmap/AggTaskV1DOMapper.xml index 3552c5b7c..47d7dc3de 100644 --- a/server/common/common-dao-gen/src/main/resources/sqlmap/AggTaskV1DOMapper.xml +++ b/server/common/common-dao-gen/src/main/resources/sqlmap/AggTaskV1DOMapper.xml @@ -7,6 +7,7 @@ + @@ -71,7 +72,7 @@ - id, gmt_create, gmt_modified, agg_id, version, deleted + id, gmt_create, gmt_modified, agg_id, version, ref_id, deleted json @@ -192,11 +193,11 @@ insert into agg_task_v1 (id, gmt_create, gmt_modified, - agg_id, version, deleted, - json) + agg_id, version, ref_id, + deleted, json) values (#{id,jdbcType=BIGINT}, #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, - #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{deleted,jdbcType=INTEGER}, - #{json,jdbcType=LONGVARCHAR}) + #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{refId,jdbcType=VARCHAR}, + #{deleted,jdbcType=INTEGER}, #{json,jdbcType=LONGVARCHAR}) insert into agg_task_v1 @@ -223,6 +224,9 @@ version, + + ref_id, + deleted, @@ -257,6 +261,9 @@ #{record.version,jdbcType=BIGINT}, + + #{record.refId,jdbcType=VARCHAR}, + #{record.deleted,jdbcType=INTEGER}, @@ -299,6 +306,9 @@ version = #{record.version,jdbcType=BIGINT}, + + ref_id = #{record.refId,jdbcType=VARCHAR}, + deleted = #{record.deleted,jdbcType=INTEGER}, @@ -319,6 +329,7 @@ gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, agg_id = #{record.aggId,jdbcType=VARCHAR}, version = #{record.version,jdbcType=BIGINT}, + ref_id = #{record.refId,jdbcType=VARCHAR}, deleted = #{record.deleted,jdbcType=INTEGER}, json = #{record.json,jdbcType=LONGVARCHAR} @@ -332,6 +343,7 @@ gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, agg_id = #{record.aggId,jdbcType=VARCHAR}, version = #{record.version,jdbcType=BIGINT}, + ref_id = #{record.refId,jdbcType=VARCHAR}, deleted = #{record.deleted,jdbcType=INTEGER} @@ -360,6 +372,9 @@ version = #{record.version,jdbcType=BIGINT}, + + ref_id = #{record.refId,jdbcType=VARCHAR}, + deleted = #{record.deleted,jdbcType=INTEGER}, @@ -377,6 +392,7 @@ gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, agg_id = #{aggId,jdbcType=VARCHAR}, version = #{version,jdbcType=BIGINT}, + ref_id = #{refId,jdbcType=VARCHAR}, deleted = #{deleted,jdbcType=INTEGER}, json = #{json,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=BIGINT} @@ -387,6 +403,7 @@ gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, agg_id = #{aggId,jdbcType=VARCHAR}, version = #{version,jdbcType=BIGINT}, + ref_id = #{refId,jdbcType=VARCHAR}, deleted = #{deleted,jdbcType=INTEGER} where id = #{id,jdbcType=BIGINT} @@ -441,12 +458,12 @@ insert into agg_task_v1 - (id, gmt_create, gmt_modified, agg_id, version, deleted, json) + (id, gmt_create, gmt_modified, agg_id, version, ref_id, deleted, json) values (#{item.id,jdbcType=BIGINT}, #{item.gmtCreate,jdbcType=TIMESTAMP}, #{item.gmtModified,jdbcType=TIMESTAMP}, - #{item.aggId,jdbcType=VARCHAR}, #{item.version,jdbcType=BIGINT}, #{item.deleted,jdbcType=INTEGER}, - #{item.json,jdbcType=LONGVARCHAR}) + #{item.aggId,jdbcType=VARCHAR}, #{item.version,jdbcType=BIGINT}, #{item.refId,jdbcType=VARCHAR}, + #{item.deleted,jdbcType=INTEGER}, #{item.json,jdbcType=LONGVARCHAR}) @@ -474,6 +491,9 @@ #{item.version,jdbcType=BIGINT} + + #{item.refId,jdbcType=VARCHAR} + #{item.deleted,jdbcType=INTEGER} @@ -509,6 +529,9 @@ version, + + ref_id, + deleted, @@ -543,6 +566,9 @@ #{record.version,jdbcType=BIGINT}, + + #{record.refId,jdbcType=VARCHAR}, + #{record.deleted,jdbcType=INTEGER}, @@ -576,6 +602,9 @@ version = #{record.version,jdbcType=BIGINT}, + + ref_id = #{record.refId,jdbcType=VARCHAR}, + deleted = #{record.deleted,jdbcType=INTEGER}, @@ -588,32 +617,34 @@ insert into agg_task_v1 - (id, gmt_create, gmt_modified, agg_id, version, deleted) + (id, gmt_create, gmt_modified, agg_id, version, ref_id, deleted) values (#{id,jdbcType=BIGINT}, #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, - #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{deleted,jdbcType=INTEGER} - ) + #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{refId,jdbcType=VARCHAR}, + #{deleted,jdbcType=INTEGER}) on duplicate key update id = #{id,jdbcType=BIGINT}, gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, agg_id = #{aggId,jdbcType=VARCHAR}, version = #{version,jdbcType=BIGINT}, + ref_id = #{refId,jdbcType=VARCHAR}, deleted = #{deleted,jdbcType=INTEGER} insert into agg_task_v1 - (id, gmt_create, gmt_modified, agg_id, version, deleted, json) + (id, gmt_create, gmt_modified, agg_id, version, ref_id, deleted, json) values (#{id,jdbcType=BIGINT}, #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, - #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{deleted,jdbcType=INTEGER}, - #{json,jdbcType=LONGVARCHAR}) + #{aggId,jdbcType=VARCHAR}, #{version,jdbcType=BIGINT}, #{refId,jdbcType=VARCHAR}, + #{deleted,jdbcType=INTEGER}, #{json,jdbcType=LONGVARCHAR}) on duplicate key update id = #{id,jdbcType=BIGINT}, gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, agg_id = #{aggId,jdbcType=VARCHAR}, version = #{version,jdbcType=BIGINT}, + ref_id = #{refId,jdbcType=VARCHAR}, deleted = #{deleted,jdbcType=INTEGER}, json = #{json,jdbcType=LONGVARCHAR} diff --git a/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/converter/MetricInfoConverter.java b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/converter/MetricInfoConverter.java index 6b031f884..6628d4e09 100644 --- a/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/converter/MetricInfoConverter.java +++ b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/converter/MetricInfoConverter.java @@ -7,6 +7,7 @@ import io.holoinsight.server.common.dao.entity.MetricInfo; import io.holoinsight.server.common.dao.entity.dto.MetricInfoDTO; import io.holoinsight.server.common.dao.transformer.ListJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import org.mapstruct.Mapper; import java.util.List; @@ -15,7 +16,7 @@ * @author jsy1001de * @version 1.0: MetricInfoConverter.java, Date: 2023-04-24 Time: 20:33 */ -@Mapper(componentModel = "spring", uses = {ListJsonMapper.class}) +@Mapper(componentModel = "spring", uses = {ListJsonMapper.class, MapJsonMapper.class}) public interface MetricInfoConverter { MetricInfoDTO doToDTO(MetricInfo metricInfo); diff --git a/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/entity/dto/MetricInfoDTO.java b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/entity/dto/MetricInfoDTO.java index 662510461..1bbd9d2cd 100644 --- a/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/entity/dto/MetricInfoDTO.java +++ b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/entity/dto/MetricInfoDTO.java @@ -8,6 +8,7 @@ import java.util.Date; import java.util.List; +import java.util.Map; /** * @author jsy1001de @@ -43,7 +44,7 @@ public class MetricInfoDTO { public boolean deleted; public String ref; - public String extInfo; + public Map extInfo; public String storageTenant; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/MapJsonMapper.java b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/transformer/MapJsonMapper.java similarity index 79% rename from server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/MapJsonMapper.java rename to server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/transformer/MapJsonMapper.java index de8d21c0d..85129096e 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/MapJsonMapper.java +++ b/server/common/common-dao/src/main/java/io/holoinsight/server/common/dao/transformer/MapJsonMapper.java @@ -1,7 +1,7 @@ /* * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. */ -package io.holoinsight.server.home.dal.transformer; +package io.holoinsight.server.common.dao.transformer; import java.lang.reflect.Type; import java.util.Map; @@ -12,7 +12,7 @@ /** * * @author jsy1001de - * @version 1.0: GaeaCollectExcutorSelectMapper.java, v 0.1 2022年03月31日 8:52 下午 jinsong.yjs Exp $ + * @version 1.0: MapJsonMapper.java, v 0.1 2022年03月31日 8:52 下午 jinsong.yjs Exp $ */ public class MapJsonMapper { public static String asString(Map map) { diff --git a/server/extension/extension-common-flyway/src/main/resources/db/migration/V23__231206_ADD_agg_task_v1_COLUMN.sql b/server/extension/extension-common-flyway/src/main/resources/db/migration/V23__231206_ADD_agg_task_v1_COLUMN.sql new file mode 100644 index 000000000..eed7b9e00 --- /dev/null +++ b/server/extension/extension-common-flyway/src/main/resources/db/migration/V23__231206_ADD_agg_task_v1_COLUMN.sql @@ -0,0 +1,9 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +-- ---------------------------- +-- Table structure for agg_task_v1 +-- ---------------------------- +ALTER TABLE `agg_task_v1` + ADD COLUMN `ref_id` VARCHAR(200) NULL DEFAULT NULL COMMENT 'ref id' AFTER `json`; diff --git a/server/home/home-common/src/main/java/io/holoinsight/server/home/common/service/query/QueryResponse.java b/server/home/home-common/src/main/java/io/holoinsight/server/home/common/service/query/QueryResponse.java index d0af5d352..b2bf7a703 100644 --- a/server/home/home-common/src/main/java/io/holoinsight/server/home/common/service/query/QueryResponse.java +++ b/server/home/home-common/src/main/java/io/holoinsight/server/home/common/service/query/QueryResponse.java @@ -10,6 +10,14 @@ @Data public class QueryResponse { + /** + * data + */ private List results; + /** + * 齐全度 + */ + private List completes; + } diff --git a/server/home/home-dal/pom.xml b/server/home/home-dal/pom.xml index 0ecee77d3..e40f1b578 100644 --- a/server/home/home-dal/pom.xml +++ b/server/home/home-dal/pom.xml @@ -47,6 +47,11 @@ registry-model ${project.version} + + ${project.groupId} + agg-core + ${project.version} + ${project.groupId} common-dao diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AggTaskV1Converter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AggTaskV1Converter.java new file mode 100644 index 000000000..0c764b155 --- /dev/null +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AggTaskV1Converter.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.dal.converter; + +import io.holoinsight.server.home.dal.model.AggTaskV1; +import io.holoinsight.server.home.dal.model.dto.AggTaskV1DTO; +import io.holoinsight.server.home.dal.transformer.AggTaskV1ConfigMapper; +import io.holoinsight.server.home.dal.transformer.BooleanIntegerMapper; +import org.mapstruct.Mapper; + +import java.util.List; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1Converter.java, Date: 2023-12-06 Time: 15:42 + */ +@Mapper(componentModel = "spring", uses = {BooleanIntegerMapper.class, AggTaskV1ConfigMapper.class}) +public interface AggTaskV1Converter { + AggTaskV1DTO doToDTO(AggTaskV1 aggTaskV1); + + AggTaskV1 dtoToDO(AggTaskV1DTO aggTaskV1DTO); + + List dosToDTOs(Iterable aggTaskV1s); +} diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlarmRuleConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlarmRuleConverter.java index 2840156a2..92fb27446 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlarmRuleConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlarmRuleConverter.java @@ -5,7 +5,7 @@ import io.holoinsight.server.common.J; import io.holoinsight.server.home.dal.model.AlarmRule; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import io.holoinsight.server.home.facade.AlarmRuleDTO; import io.holoinsight.server.home.facade.AlertRuleExtra; import org.apache.commons.lang3.StringUtils; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlertNotifyRecordConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlertNotifyRecordConverter.java index 1dbbf9e7f..aa848aaed 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlertNotifyRecordConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/AlertNotifyRecordConverter.java @@ -4,7 +4,7 @@ package io.holoinsight.server.home.dal.converter; import io.holoinsight.server.home.dal.model.AlertNotifyRecord; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import io.holoinsight.server.home.facade.AlertNotifyRecordDTO; import org.mapstruct.Mapper; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/DisplayTemplateConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/DisplayTemplateConverter.java index 3af2732a0..26ad99e5f 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/DisplayTemplateConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/DisplayTemplateConverter.java @@ -5,7 +5,7 @@ import io.holoinsight.server.home.dal.model.DisplayTemplate; import io.holoinsight.server.home.dal.model.dto.DisplayTemplateDTO; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import org.mapstruct.Mapper; import java.util.List; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigMapper.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigConverter.java similarity index 90% rename from server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigMapper.java rename to server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigConverter.java index 8e5e3f315..1f8504752 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigMapper.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/GaeaCollectConfigConverter.java @@ -8,7 +8,7 @@ import io.holoinsight.server.home.dal.transformer.BooleanIntegerMapper; import io.holoinsight.server.home.dal.transformer.GaeaCollectConfigJsonMapper; import io.holoinsight.server.home.dal.transformer.GaeaCollectRangeMapper; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import org.mapstruct.Mapper; import java.util.List; @@ -20,7 +20,7 @@ */ @Mapper(componentModel = "spring", uses = {BooleanIntegerMapper.class, GaeaCollectConfigJsonMapper.class, MapJsonMapper.class, GaeaCollectRangeMapper.class}) -public interface GaeaCollectConfigMapper { +public interface GaeaCollectConfigConverter { GaeaCollectConfigDTO doToDTO(GaeaCollectConfig gaeaCollectConfig); GaeaCollectConfig dtoToDO(GaeaCollectConfigDTO gaeaCollectConfigDTO); diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationGeneratedConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationGeneratedConverter.java index 8176cd300..833b5ae5a 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationGeneratedConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationGeneratedConverter.java @@ -5,7 +5,7 @@ import io.holoinsight.server.home.dal.model.IntegrationGenerated; import io.holoinsight.server.home.dal.model.dto.IntegrationGeneratedDTO; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import org.mapstruct.Mapper; import java.util.List; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationPluginConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationPluginConverter.java index 04e4055ab..a2b62f4b4 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationPluginConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationPluginConverter.java @@ -9,7 +9,7 @@ import io.holoinsight.server.home.dal.model.dto.IntegrationPluginDTO; import io.holoinsight.server.home.dal.transformer.GaeaCollectRangeMapper; import io.holoinsight.server.home.dal.transformer.JsonObjectMapper; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import org.apache.commons.lang3.StringUtils; import org.mapstruct.Mapper; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationProductConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationProductConverter.java index e24d20b2f..57c580292 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationProductConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/IntegrationProductConverter.java @@ -6,7 +6,7 @@ import io.holoinsight.server.home.dal.model.IntegrationProduct; import io.holoinsight.server.home.dal.model.dto.IntegrationProductDTO; import io.holoinsight.server.home.dal.transformer.FormJsonMapper; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import io.holoinsight.server.home.dal.transformer.MetricJsonMapper; import org.mapstruct.Mapper; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/UserinfoConverter.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/UserinfoConverter.java index 915ec4258..6bcdef3cd 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/UserinfoConverter.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/converter/UserinfoConverter.java @@ -4,7 +4,7 @@ package io.holoinsight.server.home.dal.converter; import io.holoinsight.server.home.dal.model.Userinfo; -import io.holoinsight.server.home.dal.transformer.MapJsonMapper; +import io.holoinsight.server.common.dao.transformer.MapJsonMapper; import io.holoinsight.server.home.facade.UserinfoDTO; import org.mapstruct.Mapper; diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/mapper/AggTaskV1Mapper.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/mapper/AggTaskV1Mapper.java new file mode 100644 index 000000000..43dbd16c5 --- /dev/null +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/mapper/AggTaskV1Mapper.java @@ -0,0 +1,15 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.dal.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import io.holoinsight.server.home.dal.model.AggTaskV1; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1Mapper.java, Date: 2023-12-06 Time: 15:39 + */ +public interface AggTaskV1Mapper extends BaseMapper { +} diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/AggTaskV1.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/AggTaskV1.java new file mode 100644 index 000000000..fba06ea7b --- /dev/null +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/AggTaskV1.java @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.dal.model; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import javax.persistence.Column; +import javax.persistence.Table; +import java.util.Date; + +/** + * @author jsy1001de + * @version 1.0: AggTask.java, Date: 2023-12-06 Time: 15:31 + */ +@Data +@Table(name = "agg_task_v1") +public class AggTaskV1 { + /** + * id + */ + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 创建时间 + */ + @Column(name = "gmt_create") + private Date gmtCreate; + + /** + * 修改时间 + */ + @Column(name = "gmt_modified") + private Date gmtModified; + + private String aggId; + + private Long version; + + private String json; + + public Integer deleted; + + public String refId; +} diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/AggTaskV1DTO.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/AggTaskV1DTO.java new file mode 100644 index 000000000..181cc14be --- /dev/null +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/AggTaskV1DTO.java @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.dal.model.dto; + +import io.holoinsight.server.agg.v1.core.conf.AggTask; +import lombok.Data; + +import java.util.Date; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1DTO.java, Date: 2023-12-06 Time: 15:34 + */ +@Data +public class AggTaskV1DTO { + /** + * id + */ + private Long id; + + /** + * 创建时间 + */ + private Date gmtCreate; + + /** + * 修改时间 + */ + private Date gmtModified; + + private String aggId; + + private Long version; + + private AggTask json; + + public Boolean deleted; + public String refId; +} diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java index 95c3c34e8..e147ebfc3 100644 --- a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java @@ -19,17 +19,18 @@ public class CollectMetric implements Serializable { private static final long serialVersionUID = -616464385087632888L; /** - * 指定存储表名称 + * 指定存储表名称, 已经废弃，暂不删除 */ + @Deprecated public String name; /** - * 监控项名称 + * 监控项名称，前端传入 */ public String tableName; /** - * 真正的指标表名称 + * 真正的指标表名称，非前端传入，是后端自动生成 */ public String targetTable; @@ -59,6 +60,17 @@ public class CollectMetric implements Serializable { // Logs that exceed 4096 are truncated public Integer sampleMaxLength = 4096; + // pre calculate, 是否开启预计算 + public Boolean calculate; + + /** + * agg 预聚合指标表名称，非前端传入，是后端自动生成 + */ + public String aggTableName; + + // 单机明细数据是否存储 + public Boolean notStorage; + @Data public static class Metric implements Serializable { diff --git a/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/AggTaskV1ConfigMapper.java b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/AggTaskV1ConfigMapper.java new file mode 100644 index 000000000..14149235a --- /dev/null +++ b/server/home/home-dal/src/main/java/io/holoinsight/server/home/dal/transformer/AggTaskV1ConfigMapper.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.dal.transformer; + +import com.google.gson.reflect.TypeToken; +import io.holoinsight.server.agg.v1.core.conf.AggTask; +import io.holoinsight.server.common.J; + +import java.lang.reflect.Type; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1ConfigMapper.java, Date: 2023-12-06 Time: 15:44 + */ +public class AggTaskV1ConfigMapper { + public static String asString(AggTask json) { + return J.toJson(json); + } + + public static AggTask asObj(String json) { + Type t = new TypeToken() {}.getType(); + return J.fromJson(json, t); + } +} diff --git a/server/home/home-service/pom.xml b/server/home/home-service/pom.xml index 9ff2e0ca2..bba31ceaf 100644 --- a/server/home/home-service/pom.xml +++ b/server/home/home-service/pom.xml @@ -92,5 +92,10 @@ extension-storage ${project.version} + + io.holoinsight.server + agg-core + ${project.version} + \ No newline at end of file diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/AggTaskUtil.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/AggTaskUtil.java new file mode 100644 index 000000000..7fd3d04df --- /dev/null +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/AggTaskUtil.java @@ -0,0 +1,167 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.biz.common; + +import io.holoinsight.server.agg.v1.core.conf.AggFunc; +import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig; +import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig.Mode; +import io.holoinsight.server.agg.v1.core.conf.From; +import io.holoinsight.server.agg.v1.core.conf.FromConfigs; +import io.holoinsight.server.agg.v1.core.conf.FromMetrics; +import io.holoinsight.server.agg.v1.core.conf.GroupBy; +import io.holoinsight.server.agg.v1.core.conf.GroupByItem; +import io.holoinsight.server.agg.v1.core.conf.Output; +import io.holoinsight.server.agg.v1.core.conf.OutputField; +import io.holoinsight.server.agg.v1.core.conf.OutputItem; +import io.holoinsight.server.agg.v1.core.conf.PartitionKey; +import io.holoinsight.server.agg.v1.core.conf.Select; +import io.holoinsight.server.agg.v1.core.conf.SelectItem; +import io.holoinsight.server.agg.v1.core.conf.SelectItem.Elect; +import io.holoinsight.server.agg.v1.core.conf.Where; +import io.holoinsight.server.agg.v1.core.conf.Window; +import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric; +import io.holoinsight.server.home.dal.model.dto.conf.CollectMetric.Metric; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * @author jsy1001de + * @version 1.0: AggTaskUtil.java, Date: 2023-12-06 Time: 14:45 + */ +public class AggTaskUtil { + + private static Integer groupByKeyLimit = 100000; + private static String OUTPUT_STORAGE_ENGINE = "TSDB"; + + public static Select buildSelect(CollectMetric collectMetric) { + + List metrics = collectMetric.getMetrics(); + + Select select = new Select(); + List items = new ArrayList<>(); + metrics.forEach(m -> { + SelectItem selectItem = new SelectItem(); + + Elect elect = new Elect(); + elect.setMetric(collectMetric.getTargetTable()); + elect.setField("value"); + selectItem.setElect(elect); + + AggFunc aggFunc = new AggFunc(); + aggFunc.setType(m.getFunc().toUpperCase()); + if (m.func.equalsIgnoreCase("count")) { + aggFunc.setType("SUM"); + } + + selectItem.setAgg(aggFunc); + + selectItem.setAs(collectMetric.getTargetTable()); + items.add(selectItem); + }); + + select.setItems(items); + return select; + } + + + public static From buildFrom(String tableName, CollectMetric collectMetric, + List groupTags) { + + From from = new From(); + from.setType("metrics"); + FromMetrics fromMetrics = new FromMetrics(); + Set metrics = new HashSet<>(); + metrics.add(collectMetric.getTargetTable()); + fromMetrics.setMetrics(metrics); + from.setMetrics(fromMetrics); + + FromConfigs fromConfigs = new FromConfigs(); + Set tableNames = new HashSet<>(); + tableNames.add(tableName); + fromConfigs.setTableNames(tableNames); + from.setConfigs(fromConfigs); + + CompletenessConfig completenessConfig = new CompletenessConfig(); + completenessConfig.setMode(Mode.COMPLETENESS_INFO); + + List groupByItems = new ArrayList<>(); + groupTags.forEach(tag -> { + GroupByItem groupByItem = new GroupByItem(); + groupByItem.setTag(tag); + groupByItem.setAs(tag); + groupByItems.add(groupByItem); + }); + GroupBy groupBy = new GroupBy(); + groupBy.setItems(groupByItems); + completenessConfig.setGroupBy(groupBy); + from.setCompleteness(completenessConfig); + return from; + } + + public static Where buildWhere(CollectMetric collectMetric) { + Where where = new Where(); + + return where; + } + + public static GroupBy buildGroupBy(CollectMetric collectMetric, List defaultGroupByTags) { + GroupBy groupBy = new GroupBy(); + groupBy.setKeyLimit(groupByKeyLimit); + + List groupByItems = new ArrayList<>(); + List tags = collectMetric.getTags(); + if (!CollectionUtils.isEmpty(defaultGroupByTags)) { + defaultGroupByTags.forEach(tag -> { + GroupByItem groupByItem = new GroupByItem(); + groupByItem.setTag(tag); + groupByItems.add(groupByItem); + }); + } + if (!CollectionUtils.isEmpty(tags)) { + tags.forEach(tag -> { + GroupByItem groupByItem = new GroupByItem(); + groupByItem.setTag(tag); + groupByItems.add(groupByItem); + }); + } + groupBy.setItems(groupByItems); + return groupBy; + } + + public static Window buildWindow(long period) { + Window window = new Window(); + window.setInterval(period); + return window; + } + + public static Output buildOutput(CollectMetric collectMetric) { + Output output = new Output(); + + OutputItem outputItem = new OutputItem(); + outputItem.setType(OUTPUT_STORAGE_ENGINE); + outputItem.setName(collectMetric.getAggTableName()); + + List fields = new ArrayList<>(); + OutputField outputField = new OutputField(); + outputField.setName(collectMetric.getTargetTable()); + outputField.setExpression(collectMetric.getTargetTable()); + fields.add(outputField); + outputItem.setFields(fields); + + List items = new ArrayList<>(); + items.add(outputItem); + + output.setItems(items); + return output; + } + + public static List buildPartition(CollectMetric collectMetric) { + return new ArrayList<>(); + } +} diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java index 54abe0992..404a65c78 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java @@ -3,14 +3,18 @@ */ package io.holoinsight.server.home.biz.listener; +import io.holoinsight.server.agg.v1.core.conf.AggTask; import io.holoinsight.server.common.dao.entity.dto.MetricInfoDTO; import io.holoinsight.server.common.service.MetricInfoService; +import io.holoinsight.server.home.biz.common.AggTaskUtil; import io.holoinsight.server.home.biz.common.GaeaConvertUtil; import io.holoinsight.server.home.biz.common.GaeaSqlTaskUtil; +import io.holoinsight.server.home.biz.service.AggTaskV1Service; import io.holoinsight.server.home.biz.service.GaeaCollectConfigService; import io.holoinsight.server.home.biz.service.TenantInitService; import io.holoinsight.server.home.common.util.EventBusHolder; import io.holoinsight.server.home.common.util.StringUtil; +import io.holoinsight.server.home.dal.model.dto.AggTaskV1DTO; import io.holoinsight.server.home.dal.model.dto.CustomPluginDTO; import io.holoinsight.server.home.dal.model.dto.CustomPluginStatus; import io.holoinsight.server.home.dal.model.dto.GaeaCollectConfigDTO; @@ -53,6 +57,9 @@ public class CustomPluginUpdateListener { @Autowired private GaeaCollectConfigService gaeaCollectConfigService; + @Autowired + private AggTaskV1Service aggTaskV1Service; + @Autowired private MetricInfoService metricInfoService; @@ -83,6 +90,7 @@ public void onEvent(CustomPluginDTO customPluginDTO) { List collectMetrics = conf.getCollectMetrics(); Map sqlTaskMaps = new HashMap<>(); + Map aggTaskMaps = new HashMap<>(); for (CollectMetric collectMetric : collectMetrics) { if (Boolean.TRUE == conf.spm && collectMetric.targetTable.contains("successPercent")) { continue; @@ -95,16 +103,24 @@ public void onEvent(CustomPluginDTO customPluginDTO) { } String tableName = String.format("%s_%s", name, customPluginDTO.id); sqlTaskMaps.put(tableName, sqlTask); + + if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.calculate) { + AggTask aggTask = buildAggTask(collectMetric, customPluginDTO); + aggTaskMaps.put(collectMetric.aggTableName, aggTask); + } } if (CollectionUtils.isEmpty(sqlTaskMaps)) return; // update - List upsert = upsert(sqlTaskMaps, customPluginDTO); + List gaeaTask = upsert(sqlTaskMaps, customPluginDTO); + List aggTask = upsertAgg(aggTaskMaps, customPluginDTO); + log.info("gaeaTask,upsert, {}", gaeaTask); + log.info("aggTask,upsert, {}", aggTask); // notify registry - notify(upsert); + notify(gaeaTask); } catch (Throwable e) { log.error("fail to convert customPlugin to gaeaCollectConfig id {} for {}", customPluginDTO.id, e.getMessage(), e); @@ -162,11 +178,6 @@ private List upsert(Map sqlTasks, CustomPluginDTO customP return upsertList; } - private void notify(List upsertList) { - - // grpc notification id update - } - private SqlTask buildSqlTask(List logPaths, CollectMetric collectMetric, CustomPluginDTO customPluginDTO) { @@ -198,58 +209,139 @@ private SqlTask buildSqlTask(List logPaths, CollectMetric collectMetric return sqlTask; } + private AggTask buildAggTask(CollectMetric collectMetric, CustomPluginDTO customPluginDTO) { + AggTask aggTask = new AggTask(); + String tableName = String.format("%s_%s", collectMetric.getTableName(), customPluginDTO.id); + aggTask.setPartitionKeys(AggTaskUtil.buildPartition(collectMetric)); + aggTask.setSelect(AggTaskUtil.buildSelect(collectMetric)); + aggTask.setFrom(AggTaskUtil.buildFrom(tableName, collectMetric, + tenantInitService.getAggCompletenessTags())); + aggTask.setWhere(AggTaskUtil.buildWhere(collectMetric)); + aggTask.setGroupBy( + AggTaskUtil.buildGroupBy(collectMetric, tenantInitService.getAggDefaultGroupByTags())); + aggTask.setWindow(AggTaskUtil.buildWindow(customPluginDTO.getPeriodType().dataUnitMs)); + aggTask.setOutput(AggTaskUtil.buildOutput(collectMetric)); + return aggTask; + } + + private List upsertAgg(Map aggTasks, CustomPluginDTO customPluginDTO) { + + if (CollectionUtils.isEmpty(aggTasks)) + return new ArrayList<>(); + + List byRefId = aggTaskV1Service.findByRefId("custom_" + customPluginDTO.getId()); + + Map byMap = new HashMap<>(); + if (!CollectionUtils.isEmpty(byRefId)) { + byRefId.forEach(by -> { + byMap.put(by.getAggId(), by); + }); + } + + List upsertList = new ArrayList<>(); + for (Map.Entry entry : aggTasks.entrySet()) { + AggTaskV1DTO aggTaskV1DTO = new AggTaskV1DTO(); + aggTaskV1DTO.setDeleted(false); + aggTaskV1DTO.setJson(entry.getValue()); + aggTaskV1DTO.setAggId(entry.getKey()); + aggTaskV1DTO.setVersion(1L); + aggTaskV1DTO.setRefId("custom_" + customPluginDTO.getId()); + + byMap.remove(entry.getKey()); + // The offline configuration is directly set to deleted=1 + if (customPluginDTO.getStatus() == CustomPluginStatus.OFFLINE) { + Long aLong = aggTaskV1Service.updateDeleted(entry.getKey()); + if (null != aLong) + upsertList.add(aLong); + continue; + } + AggTaskV1DTO upsert = aggTaskV1Service.upsert(aggTaskV1DTO); + if (null != upsert) { + upsertList.add(upsert.getId()); + } + } + + if (!CollectionUtils.isEmpty(byMap)) { + byMap.forEach((key, val) -> { + aggTaskV1Service.updateDeleted(val.getId()); + upsertList.add(val.getId()); + }); + } + + return upsertList; + } + private void saveMetricInfo(CustomPluginDTO customPluginDTO) { CustomPluginConf conf = customPluginDTO.getConf(); List collectMetrics = conf.getCollectMetrics(); for (CollectMetric collectMetric : collectMetrics) { + saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, false); + if (Boolean.TRUE == collectMetric.calculate) { + saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, true); + } + } + } + - String tableName = collectMetric.getTableName(); - if (StringUtil.isNotBlank(collectMetric.name)) { - tableName = collectMetric.getName(); + private void saveMetricByCollectMetric(CustomPluginDTO customPluginDTO, + CollectMetric collectMetric, Boolean isSpm, Boolean isAgg) { + String tableName = collectMetric.getTableName(); + String targetTable = collectMetric.getTargetTable(); + if (isAgg) { + targetTable = collectMetric.getAggTableName(); + } + + try { + MetricInfoDTO metricInfoDTO = new MetricInfoDTO(); + metricInfoDTO.setTenant(customPluginDTO.getTenant()); + metricInfoDTO.setWorkspace( + null == customPluginDTO.getWorkspace() ? "-" : customPluginDTO.getWorkspace()); + metricInfoDTO.setOrganization("-"); + metricInfoDTO.setProduct("logmonitor"); + + metricInfoDTO.setMetricType("logdefault"); + if (Boolean.TRUE == isSpm && collectMetric.targetTable.contains("successPercent")) { + metricInfoDTO.setMetricType("logspm"); + } else if (collectMetric.checkLogPattern()) { + metricInfoDTO.setMetricType("logpattern"); + } else if (collectMetric.checkLogSample()) { + metricInfoDTO.setMetricType("logsample"); + } + if (isAgg) { + Map extInfo = new HashMap<>(); + extInfo.put("isAgg", true); + metricInfoDTO.setExtInfo(extInfo); } - try { - MetricInfoDTO metricInfoDTO = new MetricInfoDTO(); - metricInfoDTO.setTenant(customPluginDTO.getTenant()); - metricInfoDTO.setWorkspace( - null == customPluginDTO.getWorkspace() ? "-" : customPluginDTO.getWorkspace()); - metricInfoDTO.setOrganization("-"); - metricInfoDTO.setProduct("logmonitor"); + metricInfoDTO.setMetric(tableName); + metricInfoDTO.setMetricTable(targetTable); + metricInfoDTO.setDeleted(customPluginDTO.status == CustomPluginStatus.OFFLINE); + metricInfoDTO.setDescription(customPluginDTO.getName() + "_" + tableName); + metricInfoDTO.setUnit("number"); + metricInfoDTO.setPeriod(customPluginDTO.getPeriodType().dataUnitMs / 1000); - metricInfoDTO.setMetricType("logdefault"); - if (Boolean.TRUE == conf.spm && collectMetric.targetTable.contains("successPercent")) { - metricInfoDTO.setMetricType("logspm"); - } else if (collectMetric.checkLogPattern()) { - metricInfoDTO.setMetricType("logpattern"); - } else if (collectMetric.checkLogSample()) { - metricInfoDTO.setMetricType("logsample"); - } - metricInfoDTO.setMetric(tableName); - metricInfoDTO.setMetricTable(collectMetric.getTargetTable()); - metricInfoDTO.setDeleted(customPluginDTO.status == CustomPluginStatus.OFFLINE); - metricInfoDTO.setDescription(customPluginDTO.getName() + "_" + tableName); - metricInfoDTO.setUnit("number"); - metricInfoDTO.setPeriod(customPluginDTO.getPeriodType().dataUnitMs / 1000); - - List tags = new ArrayList<>(Arrays.asList("ip", "hostname", "namespace")); - if (!CollectionUtils.isEmpty(collectMetric.tags)) { - tags.addAll(collectMetric.tags); - } - metricInfoDTO.setTags(tags); - metricInfoDTO.setRef(String.valueOf(customPluginDTO.getId())); - MetricInfoDTO db = metricInfoService.queryByMetric(metricInfoDTO.getTenant(), - metricInfoDTO.getWorkspace(), collectMetric.getTargetTable()); - if (null == db) { - metricInfoService.create(metricInfoDTO); - } else { - metricInfoDTO.setId(db.id); - metricInfoService.update(metricInfoDTO); - } - } catch (Exception e) { - log.error("saveLogMetricInfo error, {}, {}", collectMetric.getTargetTable(), e.getMessage(), - e); + List tags = new ArrayList<>(Arrays.asList("ip", "hostname", "namespace")); + if (!isAgg && !CollectionUtils.isEmpty(collectMetric.tags)) { + tags.addAll(collectMetric.tags); + } + metricInfoDTO.setTags(tags); + metricInfoDTO.setRef(String.valueOf(customPluginDTO.getId())); + MetricInfoDTO db = metricInfoService.queryByMetric(metricInfoDTO.getTenant(), + metricInfoDTO.getWorkspace(), targetTable); + if (null == db) { + metricInfoService.create(metricInfoDTO); + } else { + metricInfoDTO.setId(db.id); + metricInfoService.update(metricInfoDTO); } + } catch (Exception e) { + log.error("saveLogMetricInfo error, {}, {}", targetTable, e.getMessage(), e); } } + + private void notify(List upsertList) { + + // grpc notification id update + } } diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/AggTaskV1Service.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/AggTaskV1Service.java new file mode 100644 index 000000000..bf88b358c --- /dev/null +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/AggTaskV1Service.java @@ -0,0 +1,28 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.biz.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import io.holoinsight.server.home.dal.model.AggTaskV1; +import io.holoinsight.server.home.dal.model.dto.AggTaskV1DTO; + +import java.util.List; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1Service.java, Date: 2023-12-06 Time: 15:37 + */ +public interface AggTaskV1Service extends IService { + + List findByRefId(String refId); + + AggTaskV1DTO create(AggTaskV1DTO aggTaskV1DTO); + + Long updateDeleted(String aggId); + + void updateDeleted(Long id); + + AggTaskV1DTO upsert(AggTaskV1DTO aggTaskV1DTO); +} diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java index d09cc6b81..513a64fbc 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/TenantInitService.java @@ -113,4 +113,8 @@ CloudMonitorRange getCollectMonitorRange(String table, String tenant, String wor Boolean checkCustomPluginLogConfParams(String tenant, String workspace, CustomPluginDTO customPluginDTO); + List getAggCompletenessTags(); + + List getAggDefaultGroupByTags(); + } diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/AggTaskV1ServiceImpl.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/AggTaskV1ServiceImpl.java new file mode 100644 index 000000000..05e046d6a --- /dev/null +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/AggTaskV1ServiceImpl.java @@ -0,0 +1,123 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ + +package io.holoinsight.server.home.biz.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import io.holoinsight.server.common.J; +import io.holoinsight.server.common.MD5Hash; +import io.holoinsight.server.home.biz.service.AggTaskV1Service; +import io.holoinsight.server.home.dal.converter.AggTaskV1Converter; +import io.holoinsight.server.home.dal.mapper.AggTaskV1Mapper; +import io.holoinsight.server.home.dal.model.AggTaskV1; +import io.holoinsight.server.home.dal.model.dto.AggTaskV1DTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author jsy1001de + * @version 1.0: AggTaskV1ServiceImpl.java, Date: 2023-12-06 Time: 15:38 + */ +@Service +@Slf4j +public class AggTaskV1ServiceImpl extends ServiceImpl + implements AggTaskV1Service { + + @Autowired + private AggTaskV1Converter aggTaskV1Converter; + + @Override + public List findByRefId(String refId) { + + Map columnMap = new HashMap<>(); + columnMap.put("ref_id", refId); + columnMap.put("deleted", 0); + return aggTaskV1Converter.dosToDTOs(listByMap(columnMap)); + } + + public AggTaskV1DTO create(AggTaskV1DTO aggTaskV1DTO) { + aggTaskV1DTO.setGmtCreate(new Date()); + aggTaskV1DTO.setGmtModified(new Date()); + AggTaskV1 aggTaskV1 = aggTaskV1Converter.dtoToDO(aggTaskV1DTO); + + save(aggTaskV1); + return aggTaskV1Converter.doToDTO(aggTaskV1); + } + + @Override + public Long updateDeleted(String aggId) { + // 查询db里 deleted=0 的配置 + Map columnMap = new HashMap<>(); + columnMap.put("agg_id", aggId); + columnMap.put("deleted", 0); + + List byRefId = listByMap(columnMap); + + if (CollectionUtils.isEmpty(byRefId)) { + // 如果db 不存在 TableName， + return null; + } + for (AggTaskV1 aggTaskV1 : byRefId) { + aggTaskV1.setDeleted(1); + aggTaskV1.setGmtModified(new Date()); + saveOrUpdate(aggTaskV1); + } + + return byRefId.get(0).getId(); + } + + @Override + public void updateDeleted(Long id) { + AggTaskV1 byId = getById(id); + + if (null == byId) { + // 如果db 不存在， + return; + } + + byId.setDeleted(1); + byId.setGmtModified(new Date()); + saveOrUpdate(byId); + } + + @Override + public AggTaskV1DTO upsert(AggTaskV1DTO aggTaskV1DTO) { + // 查询db里 deleted=0 的配置 + Map columnMap = new HashMap<>(); + columnMap.put("agg_id", aggTaskV1DTO.getAggId()); + columnMap.put("deleted", 0); + + List byTableName = listByMap(columnMap); + + if (CollectionUtils.isEmpty(byTableName)) { + // 如果db 不存在 TableName，则新增 + return create(aggTaskV1DTO); + } + + // db 里面 agg_id+version 是组合唯一键 + // 判断是否有更新，如果没有更新则跳过 + // 如果有更新，先将 old 配置置为 deleted=1，然后重新创建一条新配置 + // md5 比对 + AggTaskV1 dbConfig = byTableName.get(0); + if (MD5Hash.getMD5(J.toJson(aggTaskV1DTO.getJson())) + .equalsIgnoreCase(MD5Hash.getMD5(dbConfig.getJson()))) { + log.info("{}-{} md5 is not update, continue", dbConfig.getId(), dbConfig.getAggId()); + return null; + } + + // 先软删除 + updateDeleted(dbConfig.getId()); + + // 然后新增 + aggTaskV1DTO.setVersion(dbConfig.getVersion() + 1); + return create(aggTaskV1DTO); + } +} diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/CustomPluginServiceImpl.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/CustomPluginServiceImpl.java index 8f5d35e83..97047b30f 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/CustomPluginServiceImpl.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/CustomPluginServiceImpl.java @@ -223,6 +223,8 @@ private CustomPluginDTO doToDTO(CustomPlugin customPlugin) { tableName = collectMetric.name; } collectMetric.targetTable = tenantInitService.getLogMonitorMetricTable(tableName); + String aggTableName = collectMetric.tableName + "_agg_" + customPluginDTO.id; + collectMetric.aggTableName = tenantInitService.getLogMonitorMetricTable(aggTableName); }); } return customPluginDTO; @@ -241,6 +243,9 @@ private List dosToDTOs(List customPlugins) { tableName = collectMetric.name; } collectMetric.targetTable = tenantInitService.getLogMonitorMetricTable(tableName); + + String aggTableName = collectMetric.tableName + "_agg_" + customPluginDTO.id; + collectMetric.aggTableName = tenantInitService.getLogMonitorMetricTable(aggTableName); }); } }); diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java index b278ba607..a15278721 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/DefaultTenantInitServiceImpl.java @@ -16,6 +16,8 @@ import io.holoinsight.server.query.grpc.QueryProto.QueryFilter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -125,4 +127,14 @@ public Boolean checkCustomPluginLogConfParams(String tenant, String workspace, return Boolean.TRUE; } + @Override + public List getAggCompletenessTags() { + return Collections.singletonList("app"); + } + + @Override + public List getAggDefaultGroupByTags() { + return new ArrayList<>(); + } + } diff --git a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/GaeaCollectConfigServiceImpl.java b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/GaeaCollectConfigServiceImpl.java index 734bc9689..45e849af3 100644 --- a/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/GaeaCollectConfigServiceImpl.java +++ b/server/home/home-service/src/main/java/io/holoinsight/server/home/biz/service/impl/GaeaCollectConfigServiceImpl.java @@ -5,6 +5,7 @@ import io.holoinsight.server.common.MD5Hash; import io.holoinsight.server.home.biz.service.GaeaCollectConfigService; +import io.holoinsight.server.home.dal.converter.GaeaCollectConfigConverter; import io.holoinsight.server.home.dal.mapper.GaeaCollectConfigMapper; import io.holoinsight.server.home.dal.model.GaeaCollectConfig; import io.holoinsight.server.home.dal.model.dto.GaeaCollectConfigDTO; @@ -30,12 +31,12 @@ public class GaeaCollectConfigServiceImpl extends ServiceImpl implements GaeaCollectConfigService { - private io.holoinsight.server.home.dal.converter.GaeaCollectConfigMapper gaeaCollectConfigMapper = - Mappers.getMapper(io.holoinsight.server.home.dal.converter.GaeaCollectConfigMapper.class); + private GaeaCollectConfigConverter gaeaCollectConfigConverter = + Mappers.getMapper(GaeaCollectConfigConverter.class); @Override public GaeaCollectConfigDTO findById(Long id) { - return gaeaCollectConfigMapper.doToDTO(getById(id)); + return gaeaCollectConfigConverter.doToDTO(getById(id)); } @Override @@ -43,7 +44,7 @@ public List findByRefId(String refId) { Map columnMap = new HashMap<>(); columnMap.put("ref_id", refId); columnMap.put("deleted", 0); - return gaeaCollectConfigMapper.dosToDTOs(listByMap(columnMap)); + return gaeaCollectConfigConverter.dosToDTOs(listByMap(columnMap)); } @Override @@ -68,8 +69,8 @@ public GaeaCollectConfigDTO upsert(GaeaCollectConfigDTO gaeaCollectConfigDTO) { // md5 比对 GaeaCollectConfig dbConfig = byTableName.get(0); - if (equelByMd5(gaeaCollectConfigMapper.dtoToDO(gaeaCollectConfigDTO), dbConfig)) { - log.info("{}-{} md5 is not update, contine", dbConfig.id, dbConfig.tableName); + if (equelByMd5(gaeaCollectConfigConverter.dtoToDO(gaeaCollectConfigDTO), dbConfig)) { + log.info("{}-{} md5 is not update, continue", dbConfig.id, dbConfig.tableName); return null; } dbConfig.setGmtModified(new Date()); @@ -107,10 +108,10 @@ private boolean equelByMd5(GaeaCollectConfig source, GaeaCollectConfig db) { public GaeaCollectConfigDTO create(GaeaCollectConfigDTO gaeaCollectConfigDTO) { gaeaCollectConfigDTO.setGmtCreate(new Date()); gaeaCollectConfigDTO.setGmtModified(new Date()); - GaeaCollectConfig gaeaCollectConfig = gaeaCollectConfigMapper.dtoToDO(gaeaCollectConfigDTO); + GaeaCollectConfig gaeaCollectConfig = gaeaCollectConfigConverter.dtoToDO(gaeaCollectConfigDTO); save(gaeaCollectConfig); - return gaeaCollectConfigMapper.doToDTO(gaeaCollectConfig); + return gaeaCollectConfigConverter.doToDTO(gaeaCollectConfig); } @Override @@ -191,9 +192,9 @@ public Long updateDeletedByRefId(String refId) { public GaeaCollectConfigDTO update(GaeaCollectConfigDTO gaeaCollectConfigDTO) { gaeaCollectConfigDTO.setGmtModified(new Date()); - GaeaCollectConfig gaeaCollectConfig = gaeaCollectConfigMapper.dtoToDO(gaeaCollectConfigDTO); + GaeaCollectConfig gaeaCollectConfig = gaeaCollectConfigConverter.dtoToDO(gaeaCollectConfigDTO); saveOrUpdate(gaeaCollectConfig); - return gaeaCollectConfigMapper.doToDTO(gaeaCollectConfig); + return gaeaCollectConfigConverter.doToDTO(gaeaCollectConfig); } } diff --git a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/AlarmSubscribeFacadeImpl.java b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/AlarmSubscribeFacadeImpl.java index 4e0d5812d..cca5f1bdb 100644 --- a/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/AlarmSubscribeFacadeImpl.java +++ b/server/home/home-web/src/main/java/io/holoinsight/server/home/web/controller/AlarmSubscribeFacadeImpl.java @@ -8,7 +8,6 @@ import java.util.Set; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import io.holoinsight.server.common.J; import io.holoinsight.server.home.common.service.RequestContextAdapter; import io.holoinsight.server.home.common.util.MonitorException; import io.holoinsight.server.home.common.util.ResultCodeEnum;