feat: save and query batch mode (#522)

* feat: batch save and query

* feat: optimize dataService init

* feat: update mocker match key

* feat: update batch query name

* feat: update batchSaveMockers name

* feat: add unit test

* feat: update comment

* fix: UT

* fix: sonar

* feat: set DataService priority by order

* fix: UT

* fix: UT

* fix: UT

* fix: UT

* fix: UT

* feat: add MergeRecordUtil comment

Author: lucas-myx

README.md

@@ -66,7 +66,7 @@ AREX utilizes the advanced Java technique, Instrument API, and is capable of ins
 - Auth0 jwt 3.x
 - JWTK jjwt 0.1+、jjwt-api 0.10+
 #### Netty
-- Netty server [3.x, 4.x]
+- Netty http server [3.x, 4.x]
 #### Config
 - Apollo Config [1.x, 2.x]
 

arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/ArexMocker.java

@@ -3,6 +3,7 @@
 import io.arex.agent.bootstrap.constants.ConfigConstants;
 import java.util.Collections;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public class ArexMocker implements Mocker {
     private String id;
@@ -15,9 +16,18 @@ public class ArexMocker implements Mocker {
     private long creationTime;
     private Mocker.Target targetRequest;
     private Mocker.Target targetResponse;
-    private boolean needMerge;
+    private transient boolean needMerge;
     private String operationName;
     private Map<String, String> tags;
+    private final transient AtomicBoolean matched = new AtomicBoolean(false);
+    /**
+     * replay match need
+     */
+    private transient int fuzzyMatchKey;
+    /**
+     * replay match need
+     */
+    private transient int accurateMatchKey;
 
     /**
      * The default constructor is for deserialization
@@ -141,4 +151,32 @@ public boolean isNeedMerge() {
     public void setNeedMerge(boolean needMerge) {
         this.needMerge = needMerge;
     }
+
+    public boolean isMatched() {
+        return matched.get();
+    }
+
+    public void setMatched(boolean matched) {
+        this.matched.compareAndSet(false, matched);
+    }
+
+    @Override
+    public int getAccurateMatchKey() {
+        return this.accurateMatchKey;
+    }
+
+    @Override
+    public void setAccurateMatchKey(int accurateMatchKey) {
+        this.accurateMatchKey = accurateMatchKey;
+    }
+
+    @Override
+    public int getFuzzyMatchKey() {
+        return this.fuzzyMatchKey;
+    }
+
+    @Override
+    public void setFuzzyMatchKey(int fuzzyMatchKey) {
+        this.fuzzyMatchKey = fuzzyMatchKey;
+    }
 }

arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/Mocker.java

@@ -117,7 +117,19 @@ default String replayLogTitle() {
         return "replay." + getCategoryType().getName();
     }
 
-    public boolean isNeedMerge();
+    boolean isNeedMerge();
 
-    public void setNeedMerge(boolean needMerge);
+    void setNeedMerge(boolean needMerge);
+
+    boolean isMatched();
+
+    void setMatched(boolean matched);
+
+    int getAccurateMatchKey();
+
+    void setAccurateMatchKey(int accurateMatchKey);
+
+    int getFuzzyMatchKey();
+
+    void setFuzzyMatchKey(int fuzzyMatchKey);
 }

arex-agent-core/src/main/java/io/arex/agent/instrumentation/BaseAgentInstaller.java

@@ -2,12 +2,10 @@
 
 import io.arex.agent.bootstrap.AgentInstaller;
 import io.arex.agent.bootstrap.TraceContextManager;
-import io.arex.agent.bootstrap.util.CollectionUtil;
 import io.arex.agent.bootstrap.util.FileUtils;
 import io.arex.foundation.config.ConfigManager;
 import io.arex.foundation.healthy.HealthManager;
 import io.arex.foundation.services.ConfigService;
-import io.arex.foundation.services.DataCollectorService;
 import io.arex.foundation.services.TimerService;
 import io.arex.foundation.util.NetUtils;
 import io.arex.inst.runtime.context.RecordLimiter;
@@ -114,15 +112,8 @@ private void initDependentComponents() {
         initDataCollector();
     }
     private void initDataCollector() {
-        DataCollector collector = DataCollectorService.INSTANCE;
-        if (ConfigManager.INSTANCE.isLocalStorage()) {
-            List<DataCollector> extendCollectorList = ServiceLoader.load(DataCollector.class, getClassLoader());
-            if (CollectionUtil.isNotEmpty(extendCollectorList)) {
-                collector = extendCollectorList.get(0);
-            }
-        }
-        collector.start();
-        DataService.builder().setDataCollector(collector).build();
+        List<DataCollector> collectorList = ServiceLoader.load(DataCollector.class, getClassLoader());
+        DataService.setDataCollector(collectorList);
     }
 
     /**

arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ArexContext.java

@@ -1,10 +1,10 @@
 package io.arex.inst.runtime.context;
 
+import io.arex.agent.bootstrap.model.Mocker;
 import io.arex.agent.bootstrap.util.ConcurrentHashSet;
 import io.arex.agent.bootstrap.util.StringUtil;
 import io.arex.inst.runtime.model.ArexConstants;
-import io.arex.inst.runtime.model.MergeDTO;
-import io.arex.inst.runtime.util.MergeRecordReplayUtil;
+import io.arex.inst.runtime.util.MergeRecordUtil;
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -18,12 +18,12 @@ public class ArexContext {
     private final long createTime;
     private volatile int sequence;
     private Set<Integer> methodSignatureHashList;
-    private Map<Integer, List<MergeDTO>> cachedReplayResultMap;
+    private Map<Integer, List<Mocker>> cachedReplayResultMap;
     private Map<String, Set<String>> excludeMockTemplate;
 
     private Map<String, Object> attachments = null;
 
-    private LinkedBlockingQueue<MergeDTO> mergeRecordQueue;
+    private LinkedBlockingQueue<Mocker> mergeRecordQueue;
 
     private boolean isRedirectRequest;
     private boolean isInvalidCase;
@@ -76,7 +76,7 @@ public Set<Integer> getMethodSignatureHashList() {
         return methodSignatureHashList;
     }
 
-    public Map<Integer, List<MergeDTO>> getCachedReplayResultMap() {
+    public Map<Integer, List<Mocker>> getCachedReplayResultMap() {
         if (cachedReplayResultMap == null) {
             cachedReplayResultMap = new ConcurrentHashMap<>();
         }
@@ -154,7 +154,7 @@ public boolean isRedirectRequest(String referer) {
         return isRedirectRequest;
     }
 
-    public LinkedBlockingQueue<MergeDTO> getMergeRecordQueue() {
+    public LinkedBlockingQueue<Mocker> getMergeRecordQueue() {
         if (mergeRecordQueue == null) {
             mergeRecordQueue = new LinkedBlockingQueue<>(2048);
         }
@@ -176,7 +176,7 @@ public void clear() {
         }
         if (mergeRecordQueue != null) {
             // async thread merge record (main entry has ended)
-            MergeRecordReplayUtil.recordRemain(this);
+            MergeRecordUtil.recordRemain(this);
             mergeRecordQueue.clear();
         }
     }

arex-instrumentation-api/src/main/java/io/arex/inst/runtime/listener/EventProcessor.java

@@ -5,6 +5,7 @@
 import io.arex.agent.bootstrap.util.AdviceClassesCollector;
 import io.arex.agent.bootstrap.util.NumberUtil;
 import io.arex.agent.bootstrap.util.StringUtil;
+import io.arex.agent.bootstrap.util.ServiceLoader;
 import io.arex.inst.runtime.model.InitializeEnum;
 import io.arex.inst.runtime.request.RequestHandlerManager;
 import io.arex.inst.runtime.log.LogManager;
@@ -14,12 +15,13 @@
 import io.arex.inst.runtime.serializer.Serializer;
 import io.arex.inst.runtime.serializer.StringSerializable;
 import io.arex.inst.runtime.util.MockUtils;
-import io.arex.agent.bootstrap.util.ServiceLoader;
 
 import java.util.List;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicReference;
+
+import io.arex.inst.runtime.util.ReplayUtil;
 import org.slf4j.LoggerFactory;
 
 public class EventProcessor {
@@ -45,6 +47,7 @@ public static void onCreate(EventSource source){
             return;
         }
         initContext(source);
+        loadReplayData();
         initClock();
         addEnterLog();
     }
@@ -134,4 +137,8 @@ private static void initLog(ClassLoader contextClassLoader) {
     public static boolean dependencyInitComplete() {
         return InitializeEnum.COMPLETE.equals(INIT_DEPENDENCY.get());
     }
+
+    private static void loadReplayData() {
+        ReplayUtil.queryMockers();
+    }
 }

arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AbstractMatchStrategy.java

@@ -1,8 +1,6 @@
 package io.arex.inst.runtime.match;
 
-import io.arex.agent.bootstrap.model.Mocker;
 import io.arex.inst.runtime.log.LogManager;
-import io.arex.inst.runtime.model.MergeDTO;
 
 public abstract class AbstractMatchStrategy {
     static final String MATCH_TITLE = "replay.match.fail";
@@ -28,15 +26,4 @@ boolean internalCheck(MatchStrategyContext context) {
         return true;
     }
     abstract void process(MatchStrategyContext context) throws Exception;
-
-    Mocker buildMatchedMocker(Mocker requestMocker, MergeDTO mergeReplayDTO) {
-        if (mergeReplayDTO == null) {
-            return null;
-        }
-        requestMocker.getTargetResponse().setBody(mergeReplayDTO.getResponse());
-        requestMocker.getTargetResponse().setType(mergeReplayDTO.getResponseType());
-        requestMocker.getTargetResponse().setAttributes(mergeReplayDTO.getResponseAttributes());
-        mergeReplayDTO.setMatched(true);
-        return requestMocker;
-    }
 }

arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AccurateMatchStrategy.java

@@ -3,51 +3,51 @@
 import io.arex.agent.bootstrap.model.MockStrategyEnum;
 import io.arex.agent.bootstrap.model.Mocker;
 import io.arex.agent.bootstrap.util.StringUtil;
-import io.arex.inst.runtime.log.LogManager;
 import io.arex.inst.runtime.model.MatchStrategyEnum;
-import io.arex.inst.runtime.model.MergeDTO;
 import io.arex.inst.runtime.util.MockUtils;
 
 import java.util.ArrayList;
 import java.util.List;
 
 public class AccurateMatchStrategy extends AbstractMatchStrategy{
-    private static final String ACCURATE_MATCH_TITLE = "replay.match.accurate";
     /**
      * search by operationName + requestBody
+     * priority:
+     * 1. if matching and not matched before return directly
+     * 2. if matched before and find-last mode, return matched one
+     * 3. if matched multiple result, give next fuzzy match
+     * 4. if strict match mode and not matched, interrupt
      */
     void process(MatchStrategyContext context) {
         context.setMatchStrategy(MatchStrategyEnum.ACCURATE);
         Mocker requestMocker = context.getRequestMocker();
-        List<MergeDTO> mergeReplayList = context.getMergeReplayList();
+        List<Mocker> replayList = context.getReplayList();
+        // operationName + requestBody
         int methodSignatureHash = MockUtils.methodSignatureHash(requestMocker);
-        List<MergeDTO> matchedList = new ArrayList<>(mergeReplayList.size());
-        for (MergeDTO mergeDTO : mergeReplayList) {
-            if (methodSignatureHash == mergeDTO.getMethodSignatureHash()) {
-                matchedList.add(mergeDTO);
+        List<Mocker> matchedList = new ArrayList<>(replayList.size());
+        for (Mocker mocker : replayList) {
+            if (methodSignatureHash == mocker.getAccurateMatchKey()) {
+                matchedList.add(mocker);
             }
         }
         int matchedCount = matchedList.size();
-        /*
-         * 1. unmatched
-         * 2. matched but find last mode (like dynamicClass)
-         */
+
         if (matchedCount == 1) {
-            if (!matchedList.get(0).isMatched() || MockStrategyEnum.FIND_LAST == context.getMockStrategy()) {
-                context.setMatchMocker(buildMatchedMocker(requestMocker, matchedList.get(0)));
+            Mocker matchMocker = matchedList.get(0);
+            // unmatched or matched but find-last mode (eg: dynamicClass)
+            if (!matchMocker.isMatched() || MockStrategyEnum.FIND_LAST == context.getMockStrategy()) {
+                matchMocker.setMatched(true);
+                context.setMatchMocker(matchMocker);
             } else {
-                LogManager.warn(ACCURATE_MATCH_TITLE, StringUtil.format("accurate match one result, but cannot be used, " +
-                                "reason: matched: %s, mock strategy: %s, methodSignatureHash: %s, category: %s",
-                                Boolean.toString(matchedList.get(0).isMatched()), context.getMockStrategy().name(),
-                                String.valueOf(methodSignatureHash), requestMocker.getCategoryType().getName()));
+                context.setReason("accurate match one result, but it has already been matched before, so cannot be used");
             }
             // other modes can only be matched once, so interrupt and not continue next fuzzy match
             context.setInterrupt(true);
             return;
         }
         // matched multiple result(like as redis: incr、decr) only retain matched item for next fuzzy match
         if (matchedCount > 1) {
-            context.setMergeReplayList(matchedList);
+            context.setReplayList(matchedList);
             return;
         }
         // if strict match mode and not matched, interrupt and not continue next fuzzy match

arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/FuzzyMatchStrategy.java

@@ -4,36 +4,41 @@
 import io.arex.agent.bootstrap.model.Mocker;
 import io.arex.agent.bootstrap.util.CollectionUtil;
 import io.arex.inst.runtime.model.MatchStrategyEnum;
-import io.arex.inst.runtime.model.MergeDTO;
 
 import java.util.List;
 
 public class FuzzyMatchStrategy extends AbstractMatchStrategy {
     /**
      * search under the same method signature
-     * @return unmatched or last one
+     * replayList is arranged in ascending order by creationTime
+     * @return not matched before or last one
      */
     void process(MatchStrategyContext context) {
         context.setMatchStrategy(MatchStrategyEnum.FUZZY);
-        Mocker requestMocker = context.getRequestMocker();
-        List<MergeDTO> mergeReplayList = context.getMergeReplayList();
-        MergeDTO matchedDTO = null;
-        int size = mergeReplayList.size();
+        List<Mocker> replayList = context.getReplayList();
+        Mocker mocker = null;
+        int size = replayList.size();
         for (int i = 0; i < size; i++) {
-            MergeDTO mergeReplayDTO = mergeReplayList.get(i);
-            if (!mergeReplayDTO.isMatched()) {
-                matchedDTO = mergeReplayDTO;
+            Mocker mockerDTO = replayList.get(i);
+            if (!mockerDTO.isMatched()) {
+                mocker = mockerDTO;
                 break;
             }
         }
-        if (matchedDTO == null && MockStrategyEnum.FIND_LAST == context.getMockStrategy()) {
-            matchedDTO = mergeReplayList.get(size - 1);
+        if (mocker == null && MockStrategyEnum.FIND_LAST == context.getMockStrategy()) {
+            mocker = replayList.get(size - 1);
         }
-        context.setMatchMocker(buildMatchedMocker(requestMocker, matchedDTO));
+        if (mocker != null) {
+            mocker.setMatched(true);
+        } else {
+            context.setReason("fuzzy match no result, all has been matched");
+        }
+
+        context.setMatchMocker(mocker);
     }
 
     @Override
     boolean internalCheck(MatchStrategyContext context) {
-        return CollectionUtil.isNotEmpty(context.getMergeReplayList());
+        return CollectionUtil.isNotEmpty(context.getReplayList());
     }
 }