Skip to content

Commit

Permalink
feat: support local desensitization (#277)
Browse files Browse the repository at this point in the history
  • Loading branch information
coryhh authored Oct 9, 2024
1 parent 2189363 commit 3a925ed
Show file tree
Hide file tree
Showing 8 changed files with 124 additions and 115 deletions.
16 changes: 16 additions & 0 deletions arex-storage-web-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,22 @@
<profile>
<build>
<finalName>storage</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>**/*.jar</exclude>
</excludes>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>**/*.yaml</include>
<include>**/*.jar</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.arextest.common.config.ConfigProvider;
import com.arextest.common.config.DefaultApplicationConfig;
import com.arextest.common.config.DefaultConfigProvider;
import com.arextest.common.desensitization.DesensitizationProvider;
import com.arextest.common.jwt.JWTService;
import com.arextest.common.jwt.JWTServiceImpl;
import com.arextest.config.model.dao.config.SystemConfigurationCollection;
Expand All @@ -13,6 +14,7 @@
import com.arextest.config.repository.impl.ApplicationServiceConfigurationRepositoryImpl;
import com.arextest.config.repository.impl.SystemConfigurationRepositoryImpl;
import com.arextest.config.utils.MongoHelper;
import com.arextest.extension.desensitization.DataDesensitization;
import com.arextest.model.mock.AREXMocker;
import com.arextest.model.mock.AREXQueryMocker;
import com.arextest.model.mock.MockCategoryType;
Expand All @@ -29,7 +31,6 @@
import com.arextest.storage.repository.RepositoryProviderFactory;
import com.arextest.storage.repository.impl.mongo.AREXMockerMongoRepositoryProvider;
import com.arextest.storage.repository.impl.mongo.AREXQueryMockerMongoRepositoryProvider;
import com.arextest.storage.repository.impl.mongo.DesensitizationLoader;
import com.arextest.storage.repository.impl.mongo.converters.ArexEigenCompressionConverter;
import com.arextest.storage.repository.impl.mongo.converters.ArexMockerCompressionConverter;
import com.arextest.storage.serialization.ZstdJacksonSerializer;
Expand All @@ -42,6 +43,8 @@
import com.arextest.storage.service.ScheduleReplayingService;
import com.arextest.storage.service.config.ApplicationService;
import com.arextest.storage.service.handler.mocker.HandleReplayResultService;
import com.arextest.storage.service.handler.mocker.coverage.CoverageEventListener;
import com.arextest.storage.service.handler.mocker.coverage.DefaultCoverageEventListener;
import com.arextest.storage.service.listener.AgentWorkingListener;
import com.arextest.storage.service.listener.AutoDiscoveryEntryPointListener;
import com.arextest.storage.web.controller.MockSourceEditionController;
Expand All @@ -58,6 +61,7 @@
import javax.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
Expand Down Expand Up @@ -115,9 +119,6 @@ public MongoDatabaseFactory mongoDbFactory() {

// todo make this optional
indexesSettingConfiguration.setIndexes(database);

// load singleton desensitization service for every thread
DesensitizationLoader.DESENSITIZATION_SERVICE = DesensitizationLoader.load(database);
syncAuthSwitch(database);
return factory;
} catch (Exception e) {
Expand All @@ -132,13 +133,26 @@ MongoTemplate mongoTemplate(MongoDatabaseFactory factory, MongoConverter convert
return new MongoTemplate(factory, converter);
}

@Bean
@ConditionalOnMissingBean(DesensitizationProvider.class)
DesensitizationProvider desensitizationProvider(MongoDatabaseFactory factory) {
String desensitizationJarUrl = DataDesensitizationUtils.getDesensitizationJarUrl(
factory.getMongoDatabase());
return new DesensitizationProvider(desensitizationJarUrl);
}

@Bean
DataDesensitization dataDesensitization(DesensitizationProvider desensitizationProvider) {
return desensitizationProvider.get();
}

@Bean
@ConditionalOnMissingBean(MongoCustomConversions.class)
public MongoCustomConversions customConversions() {
public MongoCustomConversions customConversions(DataDesensitization dataDesensitization) {
return MongoCustomConversions.create((adapter) -> {
// Type based converter
adapter.registerConverter(new ArexMockerCompressionConverter.Read());
adapter.registerConverter(new ArexMockerCompressionConverter.Write());
adapter.registerConverter(new ArexMockerCompressionConverter.Read(dataDesensitization));
adapter.registerConverter(new ArexMockerCompressionConverter.Write(dataDesensitization));

// Property based converter
adapter.configurePropertyConversions((register) -> {
Expand Down Expand Up @@ -257,7 +271,8 @@ public ScheduleReplayQueryController scheduleReplayQueryController(
CacheProvider redisCacheProvider,
DefaultApplicationConfig applicationDefaultConfig) {
return new ScheduleReplayQueryController(scheduleReplayingService, prepareMockResultService,
invalidRecordService, handleReplayResultService, redisCacheProvider, applicationDefaultConfig);
invalidRecordService, handleReplayResultService, redisCacheProvider,
applicationDefaultConfig);
}

@Bean
Expand Down Expand Up @@ -311,6 +326,12 @@ public DefaultApplicationConfig defaultApplicationConfig(ConfigProvider configPr
return new DefaultApplicationConfig(configProvider);
}

@Bean
@ConditionalOnMissingBean(CoverageEventListener.class)
public CoverageEventListener defaultCoverageEventListener() {
return new DefaultCoverageEventListener();
}

@Bean
@Order(3)
public RepositoryProvider<AREXMocker> autoPinnedMockerProvider(MongoTemplate mongoTemplate,
Expand Down Expand Up @@ -348,7 +369,8 @@ public RepositoryProvider<AREXQueryMocker> defaultQueryMockerProvider(MongoTempl
@Order(5)
public RepositoryProvider<AREXQueryMocker> pinnedQueryMockerProvider(MongoTemplate mongoTemplate,
Set<MockCategoryType> entryPointTypes, DefaultApplicationConfig defaultApplicationConfig) {
return new AREXQueryMockerMongoRepositoryProvider(ProviderNames.PINNED, mongoTemplate, properties,
return new AREXQueryMockerMongoRepositoryProvider(ProviderNames.PINNED, mongoTemplate,
properties,
entryPointTypes, defaultApplicationConfig);
}

Expand All @@ -370,4 +392,22 @@ private void syncAuthSwitch(MongoDatabase database) {
LOGGER.error("sync auth switch failed", e);
}
}

private static class DataDesensitizationUtils {

private static final String SYSTEM_CONFIGURATION = "SystemConfiguration";
private static final String DESENSITIZATION_JAR = "desensitizationJar";
private static final String JAR_URL = "jarUrl";

private static String getDesensitizationJarUrl(MongoDatabase database) {
MongoCollection<Document> collection = database.getCollection(SYSTEM_CONFIGURATION);
Bson filter = Filters.eq(SystemConfigurationCollection.Fields.key,
KeySummary.DESERIALIZATION_JAR);
Document document = collection.find(filter).first();
if (document != null && document.get(DESENSITIZATION_JAR) != null) {
return document.get(DESENSITIZATION_JAR, Document.class).getString(JAR_URL);
}
return null;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import static com.arextest.storage.model.Constants.AREX_CONFIG_MOCKERCONVERT_ENABLED_DEFAULT;

import com.arextest.common.config.DefaultApplicationConfig;
import com.arextest.diff.model.classloader.RemoteJarClassLoader;
import com.arextest.diff.utils.RemoteJarLoaderUtils;
import com.arextest.common.model.classloader.RemoteJarClassLoader;
import com.arextest.common.utils.RemoteJarLoaderUtils;
import com.arextest.extension.mockconvert.MockerConverter;
import com.arextest.model.mock.AREXMocker;
import com.arextest.model.mock.MockCategoryType;
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,51 +1,62 @@
package com.arextest.storage.repository.impl.mongo.converters;

import com.arextest.common.utils.SerializationUtils;
import com.arextest.extension.desensitization.DataDesensitization;
import com.arextest.model.mock.Mocker.Target;
import com.arextest.storage.repository.impl.mongo.DesensitizationLoader;
import com.mongodb.lang.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;

@Slf4j
public class ArexMockerCompressionConverter {
private static Target read(String source) {
String encodeWithDecryptString = null;
try {
encodeWithDecryptString = DesensitizationLoader.DESENSITIZATION_SERVICE.decrypt(source);
} catch (Exception e) {
LOGGER.error("Data decrypt failed", e);
}
return SerializationUtils.useZstdDeserialize(encodeWithDecryptString, Target.class);
}

private static String write(Target source) {
String base64Result = SerializationUtils.useZstdSerializeToBase64(source);
try {
base64Result = DesensitizationLoader.DESENSITIZATION_SERVICE.encrypt(base64Result);
return base64Result;
} catch (Exception e) {
LOGGER.error("Data encrypt failed", e);
}
return base64Result;
}

@ReadingConverter
@RequiredArgsConstructor
public static class Read implements Converter<String, Target> {

private final DataDesensitization dataDesensitization;

@Override
public Target convert(@NonNull String source) {
return read(source);
}

private Target read(String source) {
String encodeWithDecryptString = null;
try {
encodeWithDecryptString = dataDesensitization.decrypt(source);
} catch (Exception e) {
LOGGER.error("Data decrypt failed", e);
}
return SerializationUtils.useZstdDeserialize(encodeWithDecryptString, Target.class);
}
}

@WritingConverter
@RequiredArgsConstructor
public static class Write implements Converter<Target, String> {

private final DataDesensitization dataDesensitization;

@Override
public String convert(@NonNull Target source) {
return write(source);
}

private String write(Target source) {
String base64Result = SerializationUtils.useZstdSerializeToBase64(source);
try {
base64Result = dataDesensitization.encrypt(base64Result);
return base64Result;
} catch (Exception e) {
LOGGER.error("Data encrypt failed", e);
}
return base64Result;
}

}

}
Original file line number Diff line number Diff line change
@@ -1,30 +1,26 @@
package com.arextest.storage.service.handler.mocker.coverage;

import com.arextest.model.mock.Mocker;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.stereotype.Service;

/**
* @author: QizhengMo
* @date: 2024/9/18 19:00
*/
@Service
@ConditionalOnMissingBean(CoverageEventListener.class)
public class DefaultCoverageEventListener implements CoverageEventListener {

@Override
public void onBeforeNewCaseRecord(Mocker coverageMocker) {
}
@Override
public void onBeforeNewCaseRecord(Mocker coverageMocker) {
}

@Override
public void onNewCaseRecorded(Mocker coverageMocker) {
}
@Override
public void onNewCaseRecorded(Mocker coverageMocker) {
}

@Override
public void onBeforeExistingCaseRecord(Mocker coverageMocker) {
}
@Override
public void onBeforeExistingCaseRecord(Mocker coverageMocker) {
}

@Override
public void onExistingCaseRecorded(Mocker coverageMocker) {
}
@Override
public void onExistingCaseRecorded(Mocker coverageMocker) {
}
}
Binary file not shown.
Loading

0 comments on commit 3a925ed

Please sign in to comment.