The eigenvalues are recalculated through configuration

arex-storage-config/pom.xml

@@ -45,7 +45,7 @@
   <parent>
     <artifactId>arex-storage-service</artifactId>
     <groupId>com.arextest</groupId>
-    <version>1.0.47</version>
+    <version>1.0.48</version>
   </parent>
 
   <properties>

arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategory.java

@@ -0,0 +1,18 @@
+package com.arextest.config.model.vo;
+
+import lombok.Data;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * created by xinyuan_wang on 2023/12/4
+ */
+@Data
+public class QueryConfigOfCategory {
+  private String operationName;
+
+  private Set<List<String>> exclusionList;
+
+  private Set<String> ignoreNodeSet;
+}

arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java

@@ -0,0 +1,23 @@
+package com.arextest.config.model.vo;
+
+import lombok.Data;
+
+import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.NotNull;
+
+/**
+ * created by xinyuan_wang on 2023/12/4
+ */
+@Data
+public class QueryConfigOfCategoryRequest {
+
+  @NotNull
+  private String appId;
+
+  private String operationName;
+
+  private String categoryName;
+
+  @NotEmpty
+  private Boolean entryPoint;
+}

arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java

@@ -0,0 +1,11 @@
+package com.arextest.config.model.vo;
+
+import lombok.Data;
+
+/**
+ * created by xinyuan_wang on 2023/12/4
+ */
+@Data
+public class QueryConfigOfCategoryResponse {
+  private QueryConfigOfCategory body;
+}

arex-storage-model/pom.xml

@@ -7,7 +7,7 @@
   <parent>
     <artifactId>arex-storage-service</artifactId>
     <groupId>com.arextest</groupId>
-    <version>1.0.47</version>
+    <version>1.0.48</version>
   </parent>
 
   <profiles>

arex-storage-web-api/pom.xml

@@ -123,7 +123,7 @@
   <parent>
     <artifactId>arex-storage-service</artifactId>
     <groupId>com.arextest</groupId>
-    <version>1.0.47</version>
+    <version>1.0.48</version>
   </parent>
 
   <profiles>

arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java

@@ -19,14 +19,14 @@ public class EigenProcessor {
 
   public static Map<Integer, Long> calculateEigen(String body, String categoryName,
       Collection<List<String>> exclusions,
-      Collection<String> nodeNames) {
+      Collection<String> ignoreNodes) {
     EigenOptions options = EigenOptions.options();
     options.putCategoryType(categoryName);
     if (CollectionUtils.isNotEmpty(exclusions)) {
       options.putExclusions(exclusions);
     }
-    if (CollectionUtils.isNotEmpty(nodeNames)) {
-      options.putIgnoreNodes(nodeNames);
+    if (CollectionUtils.isNotEmpty(ignoreNodes)) {
+      options.putIgnoreNodes(ignoreNodes);
     }
     EigenResult eigenResult = eigenSDK.calculateEigen(body, options);
     if (eigenResult == null) {

arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java

@@ -34,5 +34,5 @@ <T extends Mocker> boolean removeRecordResult(MockCategoryType category, String
    * @param item
    * @return
    */
-  void calculateEigen(@NotNull Mocker item);
+  void calculateEigen(@NotNull Mocker item, boolean saveMocker);
 }

arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java

@@ -2,6 +2,7 @@
 
 import com.arextest.common.cache.CacheProvider;
 import com.arextest.common.utils.CompressionUtils;
+import com.arextest.config.model.vo.QueryConfigOfCategory;
 import com.arextest.model.constants.MockAttributeNames;
 import com.arextest.model.mock.AREXMocker;
 import com.arextest.model.mock.MockCategoryType;
@@ -16,10 +17,12 @@
 import com.arextest.storage.mock.internal.matchkey.impl.DatabaseMatchKeyBuilderImpl;
 import com.arextest.storage.model.MockResultType;
 import com.arextest.storage.serialization.ZstdJacksonSerializer;
+import com.arextest.storage.service.QueryConfigService;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -70,6 +73,8 @@ final class DefaultMockResultProviderImpl implements MockResultProvider {
   private MatchStrategyMetricService matchStrategyMetricService;
   @Resource
   private DatabaseMatchKeyBuilderImpl databaseMatchKeyBuilder;
+  @Resource
+  private QueryConfigService queryConfigService;
 
   /**
    * 1. Store recorded data and matching keys in redis 2. The mock type associated with dubbo, which
@@ -134,7 +139,7 @@ private <T extends Mocker> int sequencePutRecordData(MockCategoryType category,
       byte[] recordIdBytes, int size, byte[] recordKey, T value, int sequence,
       Map<byte[], Integer> mockSequenceKeyMaps) {
     if (useEigenMatch && MapUtils.isEmpty(value.getEigenMap())) {
-      calculateEigen(value);
+      calculateEigen(value, true);
     }
     List<byte[]> mockKeyList = matchKeyFactory.build(value);
     final byte[] zstdValue = serializer.serialize(value);
@@ -243,7 +248,7 @@ public <T extends Mocker> boolean removeRecordResult(MockCategoryType category,
   }
 
   @Override
-  public void calculateEigen(Mocker item) {
+  public void calculateEigen(Mocker item, boolean saveMocker) {
     try {
       if (item.getCategoryType().isEntryPoint()) {
         return;
@@ -254,8 +259,20 @@ public void calculateEigen(Mocker item) {
         return;
       }
 
+      // get exclusion and ignore node from arex-api.use this to reduction noise
+      Collection<List<String>> exclusions = null;
+      Collection<String> ignoreNodes = null;
+      if (saveMocker) {
+        QueryConfigOfCategory queryConfigOfCategory = queryConfigService.queryConfigOfCategory(
+            item);
+        if (queryConfigOfCategory != null) {
+          exclusions = queryConfigOfCategory.getExclusionList();
+          ignoreNodes = queryConfigOfCategory.getIgnoreNodeSet();
+        }
+      }
+
       Map<Integer, Long> calculateEigen = EigenProcessor.calculateEigen(eigenBody,
-          item.getCategoryType().getName(), null, null);
+          item.getCategoryType().getName(), exclusions, ignoreNodes);
       if (MapUtils.isEmpty(calculateEigen)) {
         LOGGER.warn("calculate eigen is null");
         return;
@@ -357,7 +374,7 @@ public byte[] getRecordResult(@NotNull Mocker mockItem, MockResultContext contex
     try {
       long start = System.currentTimeMillis();
       if (useEigenMatch) {
-        calculateEigen(mockItem);
+        calculateEigen(mockItem, false);
       }
       List<byte[]> mockKeyList = matchKeyFactory.build(mockItem);
       long end = System.currentTimeMillis();

arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java

@@ -68,7 +68,7 @@ public <T extends Mocker> boolean saveRecord(@NotNull T item) {
       return false;
     }
 
-    mockResultProvider.calculateEigen(item);
+    mockResultProvider.calculateEigen(item, true);
     MockerSaveHandler<T> handler = mockerHandlerFactory.getHandler(item.getCategoryType());
     if (handler != null) {
       handler.handle(item);

arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java

@@ -0,0 +1,84 @@
+package com.arextest.storage.service;
+
+import com.arextest.common.cache.CacheProvider;
+import com.arextest.config.model.vo.QueryConfigOfCategory;
+import com.arextest.config.model.vo.QueryConfigOfCategoryRequest;
+import com.arextest.config.model.vo.QueryConfigOfCategoryResponse;
+import com.arextest.model.mock.Mocker;
+import com.arextest.storage.cache.CacheKeyUtils;
+import com.arextest.storage.client.HttpWepServiceApiClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+import static com.arextest.diff.utils.JacksonHelperUtil.objectMapper;
+
+/**
+ * query config service
+ * created by xinyuan_wang on 2023/11/5
+ */
+@Service
+@Slf4j
+public class QueryConfigService {
+  private static final String CONFIG_PREFIX = "config_";
+  @Value("${arex.query.config.url}")
+  private String queryConfigOfCategoryUrl;
+  @Value("${arex.query.config.cache.expired.seconds:600}")
+  private long cacheExpiredSeconds;
+  @Resource
+  private HttpWepServiceApiClient httpWepServiceApiClient;
+  @Resource
+  private CacheProvider redisCacheProvider;
+
+  public QueryConfigOfCategory queryConfigOfCategory(Mocker mocker) {
+    if (mocker.getCategoryType().isSkipComparison()) {
+      return null;
+    }
+    QueryConfigOfCategory configCache = getConfigCache(mocker.getAppId(), mocker.getOperationName());
+    if (configCache != null) {
+      return null;
+    }
+
+    QueryConfigOfCategoryRequest queryConfigOfCategoryRequest = new QueryConfigOfCategoryRequest();
+    queryConfigOfCategoryRequest.setCategoryName(mocker.getCategoryType().getName());
+    queryConfigOfCategoryRequest.setAppId(mocker.getAppId());
+    queryConfigOfCategoryRequest.setEntryPoint(mocker.getCategoryType().isEntryPoint());
+    queryConfigOfCategoryRequest.setOperationName(mocker.getOperationName());
+    QueryConfigOfCategoryResponse queryConfigOfCategoryResponse =
+            httpWepServiceApiClient.jsonPost(queryConfigOfCategoryUrl, queryConfigOfCategoryRequest, QueryConfigOfCategoryResponse.class);
+    if (queryConfigOfCategoryResponse != null && queryConfigOfCategoryResponse.getBody() != null) {
+      putConfigCache(mocker.getAppId(), mocker.getOperationName(), queryConfigOfCategoryResponse.getBody());
+      return queryConfigOfCategoryResponse.getBody();
+    }
+    return null;
+  }
+
+  public boolean putConfigCache(String appId, String operationName, QueryConfigOfCategory response) {
+    try {
+      byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + operationName);
+      byte[] values = CacheKeyUtils.toUtf8Bytes(objectMapper.writeValueAsString(response));
+      redisCacheProvider.put(key, cacheExpiredSeconds, values);
+      return true;
+    } catch (Exception e) {
+      LOGGER.error("putConfigCache failed!", e);
+      return false;
+    }
+  }
+
+  public QueryConfigOfCategory getConfigCache(String appId, String operationName) {
+    try {
+      byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + operationName);
+      byte[] values = redisCacheProvider.get(key);
+      if (values == null) {
+        return null;
+      }
+      return objectMapper.readValue(new String(values), QueryConfigOfCategory.class);
+    } catch (Exception e) {
+      LOGGER.error("getConfigCache failed!", e);
+      return null;
+    }
+  }
+
+}

arex-storage-web-api/src/main/resources/application.yaml

@@ -12,6 +12,9 @@ arex:
   app:
     auth:
       switch: true
+  query:
+    config:
+      url: ${arex.api.service.api:http://arex-api-service:8080}
   config:
     # default for application settings
     application:

pom.xml

@@ -408,5 +408,5 @@
   <url>https://github.com/arextest/arex-storage</url>
 
 
-  <version>1.0.47</version>
+  <version>1.0.48</version>
 </project>