feat:Limit SQL length to avoid Stack Overflow

arex-storage-web-api/src/main/java/com/arextest/storage/beans/StorageAutoConfiguration.java

@@ -26,6 +26,8 @@
 import com.arextest.storage.repository.impl.mongo.converters.ArexEigenCompressionConverter;
 import com.arextest.storage.repository.impl.mongo.converters.ArexMockerCompressionConverter;
 import com.arextest.storage.serialization.ZstdJacksonSerializer;
+import com.arextest.storage.service.config.impl.ApplicationPropertiesConfigProvider;
+import com.arextest.storage.service.config.provider.ConfigProvider;
 import com.arextest.storage.service.listener.AgentWorkingListener;
 import com.arextest.storage.service.AgentWorkingService;
 import com.arextest.storage.service.listener.AutoDiscoveryEntryPointListener;
@@ -296,6 +298,13 @@ public RepositoryProvider<AREXMocker> defaultMockerProvider(MongoTemplate mongoT
     return new AREXMockerMongoRepositoryProvider(mongoTemplate, properties, entryPointTypes);
   }
 
+
+  @Bean
+  @ConditionalOnMissingBean(ConfigProvider.class)
+  public ConfigProvider applicationPropertiesConfigProvider() {
+    return new ApplicationPropertiesConfigProvider();
+  }
+
   private void syncAuthSwitch(MongoDatabase database) {
     try {
       MongoCollection<SystemConfigurationCollection> mongoCollection = database.getCollection(

arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java

@@ -18,6 +18,7 @@
 import com.arextest.storage.model.MockResultType;
 import com.arextest.storage.serialization.ZstdJacksonSerializer;
 import com.arextest.storage.service.QueryConfigService;
+import com.arextest.storage.service.config.ApplicationDefaultConfig;
 import com.arextest.storage.utils.DatabaseUtils;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -41,6 +42,8 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
+import static com.arextest.storage.model.Constants.MAX_SQL_LENGTH;
+
 
 @Component
 @Slf4j
@@ -73,6 +76,8 @@ final class DefaultMockResultProviderImpl implements MockResultProvider {
   private DubboConsumerMatchKeyBuilderImpl dubboConsumerMatchKeyBuilder;
   @Resource
   private QueryConfigService queryConfigService;
+  @Resource
+  private ApplicationDefaultConfig applicationDefaultConfig;
 
   /**
    * 1. Store recorded data and matching keys in redis 2. The mock type associated with dubbo, which
@@ -94,7 +99,7 @@ public <T extends Mocker> boolean putRecordResult(MockCategoryType category, Str
     // Obtain the number of the same interfaces in recorded data
     while (valueIterator.hasNext()) {
       T value = valueIterator.next();
-      DatabaseUtils.regenerateOperationName(value);
+      DatabaseUtils.regenerateOperationName(value, applicationDefaultConfig.getConfigAsInt(MAX_SQL_LENGTH, 50000));
       mockList.add(value);
       if (shouldRecordCallReplayMax) {
         // Dubbo type mock needs to calculate the number of body and methods combined

arex-storage-web-api/src/main/java/com/arextest/storage/mock/internal/matchkey/impl/DatabaseMatchKeyBuilderImpl.java

@@ -10,20 +10,20 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
 
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
 /**
  * Why the db mock key has more items building? 1,The user add some comments to sql or 2,add or
  * remove columns from select or 3,The value of parameter is random or 4,The sql parameter's order

arex-storage-web-api/src/main/java/com/arextest/storage/model/Constants.java

@@ -27,4 +27,7 @@ public interface Constants{
   String EXPIRATION_TIME = "expirationTime";
   // endregion
 
+  // applicationConfig
+  String MAX_SQL_LENGTH = "maxSqlLength";
+
 }

arex-storage-web-api/src/main/java/com/arextest/storage/service/config/ApplicationDefaultConfig.java

@@ -0,0 +1,32 @@
+package com.arextest.storage.service.config;
+
+import com.arextest.storage.service.config.provider.ConfigProvider;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+
+/**
+ *
+ * @author niyan
+ * @date 2024/6/6
+ * @since 1.0.0
+ */
+@Component
+@Slf4j
+public class ApplicationDefaultConfig {
+
+    @Resource(name = "applicationPropertiesConfigProvider")
+    private ConfigProvider configProvider;
+
+
+    public String getConfigAsString(String key) {
+        return configProvider.getConfigAsString(key);
+    }
+
+
+    public int getConfigAsInt(String key, int defaultValue) {
+        return configProvider.getConfigAsInt(key, defaultValue);
+    }
+
+}

arex-storage-web-api/src/main/java/com/arextest/storage/service/config/impl/ApplicationPropertiesConfigProvider.java

@@ -0,0 +1,46 @@
+package com.arextest.storage.service.config.impl;
+
+import com.arextest.storage.service.config.provider.ConfigProvider;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.core.env.Environment;
+
+import javax.annotation.Resource;
+import java.util.Map;
+
+/**
+ * @author niyan
+ * @date 2024/6/6
+ * @since 1.0.0
+ */
+@Slf4j
+public class ApplicationPropertiesConfigProvider implements ConfigProvider {
+
+    @Resource
+    private Environment environment;
+
+    @Override
+    public void loadConfigs(String configName) {
+        // nothing to do
+    }
+
+    @Override
+    public void onChange(Map<String, String> configs) {
+        // nothing to do
+    }
+
+
+    @Override
+    public String getConfigAsString(String key) {
+        return environment.getProperty(key, String.class, StringUtils.EMPTY);
+    }
+
+    @Override
+    public int getConfigAsInt(String key, int defaultValue) {
+        try {
+            return environment.getProperty(key, Integer.class, defaultValue);
+        } catch (Exception e) {
+            return defaultValue;
+        }
+    }
+}

arex-storage-web-api/src/main/java/com/arextest/storage/service/config/provider/ConfigProvider.java

@@ -0,0 +1,30 @@
+package com.arextest.storage.service.config.provider;
+
+import java.util.Map;
+
+/**
+ * @author niyan
+ * @date 2024/6/6
+ * @since 1.0.0
+ */
+public interface ConfigProvider {
+    /**
+     * load all configs
+     */
+    void loadConfigs(String configName);
+
+    /**
+     * replace all configs
+     */
+    void onChange(Map<String, String> configs);
+
+    /**
+     * get config as string
+     */
+    String getConfigAsString(String key);
+
+    /**
+     * get config as int
+     */
+    int getConfigAsInt(String key, int defaultValue);
+}

arex-storage-web-api/src/main/java/com/arextest/storage/service/handler/mocker/database/DatabaseMockerHandler.java

@@ -2,29 +2,41 @@
 
 import com.arextest.model.mock.MockCategoryType;
 import com.arextest.model.mock.Mocker;
+import com.arextest.storage.service.config.ApplicationDefaultConfig;
 import com.arextest.storage.service.handler.mocker.MockerHandler;
 import com.arextest.storage.utils.DatabaseUtils;
+import lombok.Setter;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.Resource;
+
+import static com.arextest.storage.model.Constants.MAX_SQL_LENGTH;
+
 /**
  * @author niyan
  * @date 2024/4/23
  * @since 1.0.0
  */
 @Component
 public class DatabaseMockerHandler implements MockerHandler {
+
+    @Setter(onMethod_={@Autowired})
+    private ApplicationDefaultConfig applicationDefaultConfig;
+
+
     @Override
     public MockCategoryType getMockCategoryType() {
         return MockCategoryType.DATABASE;
     }
 
     @Override
     public void handleOnRecordSaving(Mocker mocker) {
-        DatabaseUtils.regenerateOperationName(mocker);
+        DatabaseUtils.regenerateOperationName(mocker, applicationDefaultConfig.getConfigAsInt(MAX_SQL_LENGTH, 50000));
     }
 
     @Override
     public void handleOnRecordMocking(Mocker mocker) {
-        DatabaseUtils.regenerateOperationName(mocker);
+        DatabaseUtils.regenerateOperationName(mocker, applicationDefaultConfig.getConfigAsInt(MAX_SQL_LENGTH, 50000));
     }
 }

arex-storage-web-api/src/main/java/com/arextest/storage/utils/DatabaseUtils.java

@@ -8,14 +8,18 @@
 import java.util.List;
 import java.util.regex.Pattern;
 
+import com.arextest.storage.service.config.ApplicationDefaultConfig;
 import com.google.common.annotations.VisibleForTesting;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.statement.Statement;
 import net.sf.jsqlparser.statement.select.Select;
 import net.sf.jsqlparser.util.TablesNamesFinder;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.Collections;
 
@@ -30,9 +34,14 @@ public class DatabaseUtils {
     private DatabaseUtils() {
     }
 
-    private static final Pattern PATTERN = Pattern.compile("(\\s+|\"\\?\")");
+    private static final Pattern PATTERN = Pattern.compile("(\\s+|\"\\?\"|\\[|\\])");
+    private static final int MAX_SQL_LENGTH = 50000;
 
     public static void regenerateOperationName(Mocker mocker) {
+        regenerateOperationName(mocker, MAX_SQL_LENGTH);
+    }
+
+    public static void regenerateOperationName(Mocker mocker, int maxSqlLengthInt) {
         if (!MockCategoryType.DATABASE.getName().equals(mocker.getCategoryType().getName())) {
             return;
         }
@@ -44,10 +53,14 @@ public static void regenerateOperationName(Mocker mocker) {
             return;
         }
 
+        if (maxSqlLengthInt <= 0) {
+            maxSqlLengthInt = MAX_SQL_LENGTH;
+        }
+
         String[] sqls = StringUtils.split(mocker.getTargetRequest().getBody(), ";");
         List<String> operationNames = new ArrayList<>(sqls.length);
         for (String sql : sqls) {
-            TableSchema tableSchema = parse(sql);
+            TableSchema tableSchema = parse(sql, maxSqlLengthInt);
             if (tableSchema == null) {
                 continue;
             }
@@ -64,7 +77,7 @@ public static void regenerateOperationName(Mocker mocker) {
      * The operation name is generated in the format of dbName-tableNames-action-originalOperationName, eg: db1@table1,table2@select@operation1
      */
     @VisibleForTesting
-    static String regenerateOperationName(TableSchema tableSchema, String originOperationName) {
+    public static String regenerateOperationName(TableSchema tableSchema, String originOperationName) {
         return new StringBuilder(100).append(StringUtils.defaultString(tableSchema.getDbName())).append('@')
             .append(StringUtils.defaultString(StringUtils.join(tableSchema.getTableNames(), ","))).append('@')
             .append(StringUtils.defaultString(tableSchema.getAction())).append("@")
@@ -114,17 +127,27 @@ public static List<String> parseTableNames(String operationName) {
         return tableList;
     }
 
+    public static TableSchema parse(String sql) {
+        return parse(sql, MAX_SQL_LENGTH);
+    }
 
     /**
      * parse table and action from sql
      * @param sql sql
      * @return table schema info
      */
     @VisibleForTesting
-    static TableSchema parse(String sql) {
+    public static TableSchema parse(String sql, int maxSqlLengthInt) {
         if (StringUtils.isEmpty(sql)) {
+            LOGGER.warn("sql is empty");
             return null;
         }
+
+        if (sql.length() > maxSqlLengthInt) {
+            LOGGER.warn("sql length is too long, sql: {}", sql);
+            return null;
+        }
+
         TableSchema tableSchema = new TableSchema();
         try {
             sql = PATTERN.matcher(sql).replaceAll(" ");
@@ -138,7 +161,7 @@ static TableSchema parse(String sql) {
                 Collections.sort(tableNameList);
             }
             tableSchema.setTableNames(tableNameList);
-        } catch (Exception e) {
+        } catch (Throwable e) {
             LOGGER.warn("parse sql error, sql: {}", sql, e);
         }
         return tableSchema;

arex-storage-web-api/src/main/resources/application.yaml

@@ -47,3 +47,4 @@ arex:
           metric: false
 pom:
   version: ${project.version}
+maxSqlLength: 50000

arex-storage-web-api/src/test/java/com/arextest/storage/utils/DatabaseUtilsTest.java

@@ -2,12 +2,15 @@
 
 import com.arextest.diff.handler.parse.sqlparse.constants.DbParseConstants;
 import com.arextest.storage.model.TableSchema;
+import com.arextest.storage.service.config.ApplicationDefaultConfig;
 import net.sf.jsqlparser.statement.upsert.Upsert;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;