The eigenvalues are recalculated through configuration (#125)

pangdayuan1 · web-flow · commit 7ff3935b7111 · 2023-12-12T16:49:22.000+08:00
* feat: The eigenvalues are recalculated through configuration
diff --git a/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java
@@ -0,0 +1,23 @@
+package com.arextest.config.model.vo;
+
+import lombok.Data;
+
+import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.NotNull;
+
+/**
+ * created by xinyuan_wang on 2023/12/4
+ */
+@Data
+public class QueryConfigOfCategoryRequest {
+
+  @NotNull
+  private String appId;
+
+  private String operationName;
+
+  private String categoryName;
+
+  @NotEmpty
+  private Boolean entryPoint;
+}
diff --git a/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java
@@ -0,0 +1,22 @@
+package com.arextest.config.model.vo;
+
+import java.util.List;
+import java.util.Set;
+import lombok.Data;
+
+/**
+ * created by xinyuan_wang on 2023/12/4
+ */
+@Data
+public class QueryConfigOfCategoryResponse {
+  private QueryConfigOfCategory body;
+
+  @Data
+  public static class QueryConfigOfCategory {
+    private String operationName;
+
+    private Set<List<String>> exclusionList;
+
+    private Set<String> ignoreNodeSet;
+  }
+}
diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java
@@ -19,14 +19,14 @@ public class EigenProcessor {
 
   public static Map<Integer, Long> calculateEigen(String body, String categoryName,
       Collection<List<String>> exclusions,
-      Collection<String> nodeNames) {
+      Collection<String> ignoreNodes) {
     EigenOptions options = EigenOptions.options();
     options.putCategoryType(categoryName);
     if (CollectionUtils.isNotEmpty(exclusions)) {
       options.putExclusions(exclusions);
     }
-    if (CollectionUtils.isNotEmpty(nodeNames)) {
-      options.putIgnoreNodes(nodeNames);
+    if (CollectionUtils.isNotEmpty(ignoreNodes)) {
+      options.putIgnoreNodes(ignoreNodes);
     }
     EigenResult eigenResult = eigenSDK.calculateEigen(body, options);
     if (eigenResult == null) {
diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java
@@ -34,5 +34,5 @@ <T extends Mocker> boolean removeRecordResult(MockCategoryType category, String
    * @param item
    * @return
    */
-  void calculateEigen(@NotNull Mocker item);
+  void calculateEigen(@NotNull Mocker item, boolean queryConfig);
 }
diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java
@@ -2,6 +2,7 @@
 
 import com.arextest.common.cache.CacheProvider;
 import com.arextest.common.utils.CompressionUtils;
+import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory;
 import com.arextest.model.constants.MockAttributeNames;
 import com.arextest.model.mock.AREXMocker;
 import com.arextest.model.mock.MockCategoryType;
@@ -16,10 +17,12 @@
 import com.arextest.storage.mock.internal.matchkey.impl.DatabaseMatchKeyBuilderImpl;
 import com.arextest.storage.model.MockResultType;
 import com.arextest.storage.serialization.ZstdJacksonSerializer;
+import com.arextest.storage.service.QueryConfigService;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -70,6 +73,8 @@ final class DefaultMockResultProviderImpl implements MockResultProvider {
   private MatchStrategyMetricService matchStrategyMetricService;
   @Resource
   private DatabaseMatchKeyBuilderImpl databaseMatchKeyBuilder;
+  @Resource
+  private QueryConfigService queryConfigService;
 
   /**
    * 1. Store recorded data and matching keys in redis 2. The mock type associated with dubbo, which
@@ -134,7 +139,7 @@ private <T extends Mocker> int sequencePutRecordData(MockCategoryType category,
       byte[] recordIdBytes, int size, byte[] recordKey, T value, int sequence,
       Map<byte[], Integer> mockSequenceKeyMaps) {
     if (useEigenMatch && MapUtils.isEmpty(value.getEigenMap())) {
-      calculateEigen(value);
+      calculateEigen(value, true);
     }
     List<byte[]> mockKeyList = matchKeyFactory.build(value);
     final byte[] zstdValue = serializer.serialize(value);
@@ -243,7 +248,7 @@ public <T extends Mocker> boolean removeRecordResult(MockCategoryType category,
   }
 
   @Override
-  public void calculateEigen(Mocker item) {
+  public void calculateEigen(Mocker item, boolean queryConfig) {
     try {
       if (item.getCategoryType().isEntryPoint()) {
         return;
@@ -254,8 +259,20 @@ public void calculateEigen(Mocker item) {
         return;
       }
 
+      // get exclusion and ignore node from arex-api.use this to reduction noise
+      Collection<List<String>> exclusions = null;
+      Collection<String> ignoreNodes = null;
+      if (queryConfig) {
+        QueryConfigOfCategory queryConfigOfCategory = queryConfigService.queryConfigOfCategory(
+            item);
+        if (queryConfigOfCategory != null) {
+          exclusions = queryConfigOfCategory.getExclusionList();
+          ignoreNodes = queryConfigOfCategory.getIgnoreNodeSet();
+        }
+      }
+
       Map<Integer, Long> calculateEigen = EigenProcessor.calculateEigen(eigenBody,
-          item.getCategoryType().getName(), null, null);
+          item.getCategoryType().getName(), exclusions, ignoreNodes);
       if (MapUtils.isEmpty(calculateEigen)) {
         LOGGER.warn("calculate eigen is null");
         return;
@@ -357,7 +374,7 @@ public byte[] getRecordResult(@NotNull Mocker mockItem, MockResultContext contex
     try {
       long start = System.currentTimeMillis();
       if (useEigenMatch) {
-        calculateEigen(mockItem);
+        calculateEigen(mockItem, false);
       }
       List<byte[]> mockKeyList = matchKeyFactory.build(mockItem);
       long end = System.currentTimeMillis();
diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java b/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java
@@ -79,7 +79,7 @@ public <T extends Mocker> boolean saveRecord(@NotNull T item) {
       return true;
     }
 
-    mockResultProvider.calculateEigen(item);
+    mockResultProvider.calculateEigen(item, true);
     RepositoryProvider<T> repositoryWriter = repositoryProviderFactory.defaultProvider();
     return repositoryWriter != null && repositoryWriter.save(item);
   }
diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java b/arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java
@@ -0,0 +1,90 @@
+package com.arextest.storage.service;
+
+import com.arextest.common.cache.CacheProvider;
+import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory;
+import com.arextest.config.model.vo.QueryConfigOfCategoryRequest;
+import com.arextest.config.model.vo.QueryConfigOfCategoryResponse;
+import com.arextest.model.mock.Mocker;
+import com.arextest.storage.cache.CacheKeyUtils;
+import com.arextest.storage.client.HttpWepServiceApiClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+import static com.arextest.diff.utils.JacksonHelperUtil.objectMapper;
+
+/**
+ * query config service
+ * created by xinyuan_wang on 2023/11/5
+ */
+@Service
+@Slf4j
+public class QueryConfigService {
+  private static final String CONFIG_PREFIX = "config_";
+  @Value("${arex.query.config.url}")
+  private String queryConfigOfCategoryUrl;
+  @Value("${arex.query.config.cache.expired.seconds:600}")
+  private long cacheExpiredSeconds;
+  @Resource
+  private HttpWepServiceApiClient httpWepServiceApiClient;
+  @Resource
+  private CacheProvider redisCacheProvider;
+
+  public QueryConfigOfCategory queryConfigOfCategory(Mocker mocker) {
+    if (mocker.getCategoryType().isSkipComparison()) {
+      return null;
+    }
+    String categoryName = mocker.getCategoryType().getName();
+    String appId = mocker.getAppId();
+    String operationName = mocker.getOperationName();
+
+    QueryConfigOfCategory configCache = getConfigCache(appId, categoryName,
+        operationName);
+    if (configCache != null) {
+      return configCache;
+    }
+
+    QueryConfigOfCategoryRequest queryConfigOfCategoryRequest = new QueryConfigOfCategoryRequest();
+    queryConfigOfCategoryRequest.setCategoryName(categoryName);
+    queryConfigOfCategoryRequest.setAppId(appId);
+    queryConfigOfCategoryRequest.setEntryPoint(mocker.getCategoryType().isEntryPoint());
+    queryConfigOfCategoryRequest.setOperationName(operationName);
+    QueryConfigOfCategoryResponse queryConfigOfCategoryResponse =
+            httpWepServiceApiClient.jsonPost(queryConfigOfCategoryUrl, queryConfigOfCategoryRequest, QueryConfigOfCategoryResponse.class);
+    if (queryConfigOfCategoryResponse != null && queryConfigOfCategoryResponse.getBody() != null) {
+      putConfigCache(appId, categoryName, operationName, queryConfigOfCategoryResponse.getBody());
+      return queryConfigOfCategoryResponse.getBody();
+    }
+    return null;
+  }
+
+  public boolean putConfigCache(String appId, String categoryName, String operationName,
+      QueryConfigOfCategory response) {
+    try {
+      byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName);
+      byte[] values = CacheKeyUtils.toUtf8Bytes(objectMapper.writeValueAsString(response));
+      redisCacheProvider.put(key, cacheExpiredSeconds, values);
+      return true;
+    } catch (Exception e) {
+      LOGGER.error("putConfigCache failed!", e);
+      return false;
+    }
+  }
+
+  public QueryConfigOfCategory getConfigCache(String appId, String categoryName, String operationName) {
+    try {
+      byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName);
+      byte[] values = redisCacheProvider.get(key);
+      if (values == null) {
+        return null;
+      }
+      return objectMapper.readValue(new String(values), QueryConfigOfCategory.class);
+    } catch (Exception e) {
+      LOGGER.error("getConfigCache failed!", e);
+      return null;
+    }
+  }
+
+}
diff --git a/arex-storage-web-api/src/main/resources/application.yaml b/arex-storage-web-api/src/main/resources/application.yaml
@@ -12,6 +12,9 @@ arex:
   app:
     auth:
       switch: true
+  query:
+    config:
+      url: ${arex.api.service.api:http://arex-api-service:8080}/api/config/comparison/summary/queryConfigOfCategory
   config:
     # default for application settings
     application:

Original file line number	Diff line number	Diff line change
`@@ -34,5 +34,5 @@ <T extends Mocker> boolean removeRecordResult(MockCategoryType category, String`
`34`	`34`	`* @param item`
`35`	`35`	`* @return`
`36`	`36`	`*/`
`37`		`- void calculateEigen(@NotNull Mocker item);`
	`37`	`+ void calculateEigen(@NotNull Mocker item, boolean queryConfig);`
`38`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,7 @@ public <T extends Mocker> boolean saveRecord(@NotNull T item) {`
`79`	`79`	`return true;`
`80`	`80`	`}`
`81`	`81`
`82`		`- mockResultProvider.calculateEigen(item);`
	`82`	`+ mockResultProvider.calculateEigen(item, true);`
`83`	`83`	`RepositoryProvider<T> repositoryWriter = repositoryProviderFactory.defaultProvider();`
`84`	`84`	`return repositoryWriter != null && repositoryWriter.save(item);`
`85`	`85`	`}`