Skip to content

Commit 7570c45

Browse files
Willam2004Willam2004
committed
feat: 0.5.14 merge
1 parent 2dcd64a commit 7570c45

File tree

65 files changed

+4663
-113
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+4663
-113
lines changed

‎CHANGELOG

+21
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,27 @@ Transformer
149149
• 修复列类型解析的bug
150150

151151

152+
向下兼容（compatibility）
153+
• 暂无
154+
155+
过期功能（deprecations）
156+
无
157+
安全漏洞修复（vulnerability）
158+
无
159+
160+
## 0.5.14
161+
新功能（new features）
162+
Core
163+
• 无
164+
165+
166+
Transformer
167+
• 新增FML到Flink建表转换
168+
• 修复Hologres的Decimal类型无法识别问题
169+
• 修复Adb Mysql的Decimal类型无法识别问题
170+
171+
172+
152173
向下兼容（compatibility）
153174
• 暂无
154175

‎docs/zh-cn/transformer/guide.md

+13-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,19 @@
6262
<groupId>com.aliyun.fastmodel</groupId>
6363
<artifactId>fastmodel-transform-hologres</artifactId>
6464
</dependency>
65-
65+
66+
<!--支持adb mysql转换-->
67+
<dependency>
68+
<groupId>com.aliyun.fastmodel</groupId>
69+
<artifactId>fastmodel-transform-adbmysql</artifactId>
70+
</dependency>
71+
72+
73+
<!--支持flink转换-->
74+
<dependency>
75+
<groupId>com.aliyun.fastmodel</groupId>
76+
<artifactId>fastmodel-transform-flink</artifactId>
77+
</dependency>
6678

6779
</dependencies>
6880
</project>

‎docs/zh-cn/transformer/maxcompute.md

Whitespace-only changes.

‎fastmodel-bom/pom.xml

+5-1
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,11 @@
189189
<artifactId>fastmodel-transform-plantuml</artifactId>
190190
<version>${project.version}</version>
191191
</dependency>
192-
192+
<dependency>
193+
<groupId>com.aliyun.fastmodel</groupId>
194+
<artifactId>fastmodel-transform-flink</artifactId>
195+
<version>${project.version}</version>
196+
</dependency>
193197

194198
</dependencies>
195199
</dependencyManagement>

‎fastmodel-converter/fastmodel-converter-dqc/src/test/java/com/aliyun/fastmodel/dqc/check/ChangeColConverterTest.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public void testConverterCodeTable() {
150150
List<ChangeDqcRuleElement> changeDqcRuleElement = changeRules.getChangeDqcRuleElement();
151151
assertEquals(changeRules.toString(), "ALTER DQC_RULE ON TABLE dim_shop\n"
152152
+ "CHANGE (\n"
153-
+ " c1 CONSTRAINT `字段规则-表中-(c1)` CHECK(IN_TABLE(c1, ``, code) = 0) NOT ENFORCED DISABLE\n"
153+
+ " c1 CONSTRAINT `字段规则-标准代码-(c1)` CHECK(IN_TABLE(c1, ``, code) = 0) NOT ENFORCED DISABLE\n"
154154
+ ")");
155155
}
156156

@@ -178,8 +178,8 @@ public void testConverterCodeTableWithOld() {
178178
List<ChangeDqcRuleElement> changeDqcRuleElement = changeRules.getChangeDqcRuleElement();
179179
assertEquals(changeRules.toString(), "ALTER DQC_RULE ON TABLE dim_shop\n"
180180
+ "CHANGE (\n"
181-
+ " c1 CONSTRAINT `字段规则-表中-(c1)` CHECK(IN_TABLE(c1, dce, code) = 0) NOT ENFORCED DISABLE,\n"
182-
+ " c1 CONSTRAINT `字段规则-表中-(c1)` CHECK(IN_TABLE(c1, abc, code) = 0) NOT ENFORCED\n"
181+
+ " c1 CONSTRAINT `字段规则-标准代码-(c1)` CHECK(IN_TABLE(c1, dce, code) = 0) NOT ENFORCED DISABLE,\n"
182+
+ " c1 CONSTRAINT `字段规则-标准代码-(c1)` CHECK(IN_TABLE(c1, abc, code) = 0) NOT ENFORCED\n"
183183
+ ")");
184184
}
185185

@@ -196,7 +196,7 @@ public void testConverterCodeTableExist() {
196196
List<ChangeDqcRuleElement> changeDqcRuleElement = changeRules.getChangeDqcRuleElement();
197197
assertEquals(changeRules.toString(), "ALTER DQC_RULE ON TABLE dim_shop\n"
198198
+ "CHANGE (\n"
199-
+ " c1 CONSTRAINT `字段规则-表中-(c1)` CHECK(IN_TABLE(c1, t1, code) = 0) NOT ENFORCED\n"
199+
+ " c1 CONSTRAINT `字段规则-标准代码-(c1)` CHECK(IN_TABLE(c1, t1, code) = 0) NOT ENFORCED\n"
200200
+ ")");
201201
}
202202
}

‎fastmodel-converter/fastmodel-converter-dqc/src/test/java/com/aliyun/fastmodel/dqc/check/DropTableConverterTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void testDropWithBefore() {
109109
assertEquals(convert.toString(), "ALTER DQC_RULE ON TABLE test_test PARTITION (ds='[a-zA-Z0-9_-]*')\n"
110110
+ "ADD (\n"
111111
+ " CONSTRAINT `字段规则-非空-(a)` CHECK(NULL_COUNT(a) = 0) NOT ENFORCED DISABLE,\n"
112-
+ " CONSTRAINT `字段规则-表中-(a)` CHECK(IN_TABLE(a, mode, code) = 0) NOT ENFORCED DISABLE,\n"
112+
+ " CONSTRAINT `字段规则-标准代码-(a)` CHECK(IN_TABLE(a, mode, code) = 0) NOT ENFORCED DISABLE,\n"
113113
+ " CONSTRAINT `字段规则-非空-(e)` CHECK(NULL_COUNT(e) = 0) NOT ENFORCED DISABLE,\n"
114114
+ " CONSTRAINT `字段规则-非空-(b)` CHECK(NULL_COUNT(b) = 0) NOT ENFORCED DISABLE,\n"
115115
+ " CONSTRAINT `字段规则-非空-(cc)` CHECK(NULL_COUNT(cc) = 0) NOT ENFORCED DISABLE,\n"

‎fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/rule/function/BaseFunctionName.java

+27-12
Original file line numberDiff line numberDiff line change
@@ -39,57 +39,72 @@ public enum BaseFunctionName {
3939
* 平均
4040
*/
4141
AVG(FunctionGrade.COLUMN, "平均值", ""),
42+
4243
/**
4344
* 汇总值
4445
*/
4546
SUM(FunctionGrade.COLUMN, "汇总值", ""),
47+
4648
/**
4749
* 空值个数
4850
*/
4951
NULL_COUNT(FunctionGrade.COLUMN, "空值数", "非空"),
5052

51-
/**
52-
* 空值/总行数
53-
*/
54-
NULL_RATE(FunctionGrade.COLUMN, "空值率", ""),
55-
5653
/**
5754
* 重复值个数
5855
*/
5956
DUPLICATE_COUNT(FunctionGrade.COLUMN, "重复值数", "唯一"),
6057

6158
/**
62-
* 重复值/总行数
59+
* 不同值个数
6360
*/
64-
DUPLICATE_RATE(FunctionGrade.COLUMN, "重复率", ""),
61+
UNIQUE_COUNT(FunctionGrade.COLUMN, "唯一值数", ""),
62+
6563
/**
66-
* 唯一值
64+
* 重复值/总行数
6765
*/
68-
UNIQUE_COUNT(FunctionGrade.COLUMN, "唯一值数", ""),
66+
DUPLICATE_PERCENT(FunctionGrade.COLUMN, "重复率", ""),
67+
6968
/**
7069
* 离散值分组数
7170
*/
72-
GROUP_COUNT(FunctionGrade.COLUMN, "离散值分组数", ""),
71+
DISCRETE_GROUP_COUNT(FunctionGrade.COLUMN, "离散值分组数", ""),
72+
7373
/**
7474
* 离散值状态值
7575
*/
76-
STATE_COUNT(FunctionGrade.COLUMN, "离散值状态数", ""),
76+
DISCRETE_VALUE_COUNT(FunctionGrade.COLUMN, "离散值状态数", ""),
7777

7878
/**
7979
* 值是否在表里
8080
*/
81-
IN_TABLE(FunctionGrade.COLUMN, "表中", ""),
81+
IN_TABLE(FunctionGrade.COLUMN, "标准代码", ""),
8282

8383
/**
8484
* 表行数
8585
*/
8686
TABLE_COUNT(FunctionGrade.TABLE, "表行数", ""),
8787

88+
/**
89+
* 表不同行数
90+
*/
91+
DISTINCT_COUNT(FunctionGrade.TABLE, "表不同行数", ""),
92+
8893
/**
8994
* 表大小
9095
*/
9196
TABLE_SIZE(FunctionGrade.TABLE, "表大小", ""),
9297

98+
/**
99+
* 表大小波动
100+
*/
101+
TABLE_SIZE_DELTA(FunctionGrade.TABLE, "表大小波动", ""),
102+
103+
/**
104+
* 表行数波动
105+
*/
106+
TABLE_COUNT_DELTA(FunctionGrade.TABLE, "表行数波动", ""),
107+
93108
/**
94109
* 表中的唯一键
95110
*/

‎fastmodel-dependencies-bom/pom.xml

+13-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
<version>${revision}</version>
2727

2828
<properties>
29-
<revision>0.5.13</revision>
29+
<revision>0.5.14</revision>
3030
<maven_flatten_version>1.1.0</maven_flatten_version>
3131
<dep.jline.version>3.17.1</dep.jline.version>
3232
<dep.antlr.version>4.13.1</dep.antlr.version>
@@ -44,7 +44,7 @@
4444
<profile>
4545
<id>jdk11</id>
4646
<properties>
47-
<revision>0.5.13</revision>
47+
<revision>0.5.14</revision>
4848
</properties>
4949
<activation>
5050
<activeByDefault>true</activeByDefault>
@@ -62,7 +62,7 @@
6262
<profile>
6363
<id>jdk8</id>
6464
<properties>
65-
<revision>0.5.13-jdk8</revision>
65+
<revision>0.5.14-jdk8</revision>
6666
</properties>
6767
<dependencyManagement>
6868
<dependencies>
@@ -297,6 +297,16 @@
297297
<scope>test</scope>
298298
</dependency>
299299

300+
<dependency>
301+
<groupId>org.apache.flink</groupId>
302+
<artifactId>flink-sql-parser</artifactId>
303+
<version>1.19.0</version>
304+
</dependency>
305+
<dependency>
306+
<groupId>org.apache.calcite</groupId>
307+
<artifactId>calcite-core</artifactId>
308+
<version>1.37.0</version>
309+
</dependency>
300310

301311
</dependencies>
302312
</dependencyManagement>

‎fastmodel-transform/fastmodel-transform-adbmysql/src/main/java/com/aliyun/fastmodel/transform/adbmysql/parser/AdbMysqlAstBuilder.java

+3
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,9 @@ public Node visitDimensionDataType(DimensionDataTypeContext ctx) {
390390
} else if (ctx.lengthTwoDimension() != null) {
391391
dataTypeParameterList = ParserHelper.visit(this, ctx.lengthTwoDimension().decimalLiteral(), DecimalLiteral.class).stream()
392392
.map(d -> new NumericParameter(d.getNumber())).collect(Collectors.toList());
393+
} else if (ctx.lengthTwoOptionalDimension() != null) {
394+
dataTypeParameterList = ParserHelper.visit(this, ctx.lengthTwoOptionalDimension().decimalLiteral(), DecimalLiteral.class).stream()
395+
.map(d -> new NumericParameter(d.getNumber())).collect(Collectors.toList());
393396
}
394397
return new GenericDataType(new Identifier(typeName.getText()), dataTypeParameterList);
395398
}

‎fastmodel-transform/fastmodel-transform-adbmysql/src/test/java/com/aliyun/fastmodel/transform/adbmysql/client/AdbMysqlGeneratorClientTest.java

+27-1
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,33 @@ public void testGeneratorDatabase() {
277277
}
278278

279279
@Test
280-
public void testGeneratorPartitionByHash() {
280+
public void testGeneratorDecimal() {
281+
List<Column> columns = Lists.newArrayList();
282+
Column c = Column.builder()
283+
.name("c1")
284+
.nullable(true)
285+
.dataType("decimal")
286+
.precision(10)
287+
.scale(10)
288+
.comment("comment")
289+
.build();
290+
columns.add(c);
291+
List<Constraint> constraints = Lists.newArrayList();
292+
List<BaseClientProperty> properties = Lists.newArrayList();
293+
TablePartitionRaw tablePartitionRaw = new TablePartitionRaw();
294+
tablePartitionRaw.setValueString("PARTITION BY VALUE(date_format(c1, '%Y%M%d'))");
295+
properties.add(tablePartitionRaw);
296+
Table table = Table.builder().name("t1")
297+
.columns(columns)
298+
.constraints(constraints)
299+
.properties(properties)
300+
.lifecycleSeconds(1000000L)
301+
.build();
302+
assertText("CREATE TABLE IF NOT EXISTS t1\n"
303+
+ "(\n"
304+
+ " c1 DECIMAL(10,10) NULL COMMENT 'comment'\n"
305+
+ ")\n"
306+
+ "PARTITION BY VALUE(date_format(c1, '%Y%M%d')) LIFECYCLE 11", table);
281307

282308
}
283309

‎fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/converter/BaseClientConverter.java

+15-2
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ protected String formatExpression(BaseExpression baseExpression) {
155155
*/
156156
@Override
157157
public Node convertToNode(Table table, TableConfig tableConfig) {
158-
QualifiedName of = StringJoinUtil.join(table.getDatabase(), table.getSchema(), table.getName());
158+
QualifiedName of = StringJoinUtil.join(table.getCatalog(), table.getDatabase(), table.getSchema(), table.getName());
159159
Comment comment = null;
160160
if (table.getComment() != null) {
161161
comment = new Comment(table.getComment());
@@ -190,6 +190,7 @@ public Table convertToTable(Node table, T context) {
190190
Preconditions.checkArgument(table instanceof CreateTable, "unsupported convert to table:" + table.getClass());
191191
CreateTable createTable = (CreateTable)table;
192192
Boolean external = isExternal(createTable);
193+
String catalog = toCatalog(createTable, context);
193194
String schema = toSchema(createTable, context);
194195
String suffix = createTable.getQualifiedName().getSuffix();
195196
String database = toDatabase(createTable, context.getDatabase());
@@ -200,6 +201,7 @@ public Table convertToTable(Node table, T context) {
200201
return Table.builder()
201202
.ifNotExist(createTable.isNotExists())
202203
.external(external)
204+
.catalog(catalog)
203205
.database(database)
204206
.schema(schema).name(suffix)
205207
.comment(createTable.getCommentValue())
@@ -330,6 +332,17 @@ protected String toSchema(CreateTable createTable, T transformContext) {
330332
return transformContext.getSchema();
331333
}
332334

335+
/**
336+
* get catalog
337+
*
338+
* @param createTable
339+
* @param transformContext
340+
* @return
341+
*/
342+
protected String toCatalog(CreateTable createTable, T transformContext) {
343+
return null;
344+
}
345+
333346
/**
334347
* get database
335348
*
@@ -627,7 +640,7 @@ protected List<BaseConstraint> toConstraint(List<Column> columns, List<Constrain
627640
return constraintList;
628641
}
629642

630-
private void setOutlineConstraint(Constraint c, List<BaseConstraint> constraintList) {
643+
protected void setOutlineConstraint(Constraint c, List<BaseConstraint> constraintList) {
631644
Identifier constraintName;
632645
if (StringUtils.isBlank(c.getName())) {
633646
constraintName = IdentifierUtil.sysIdentifier();

‎fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/dto/table/Table.java

+5
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ public class Table {
3737
*/
3838
private boolean external;
3939

40+
/**
41+
* catalog名称
42+
*/
43+
private String catalog;
44+
4045
/**
4146
* database 名字
4247
*/

‎fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/context/TransformContext.java

+17
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ public class TransformContext {
4242
*/
4343
private boolean appendSemicolon;
4444

45+
/**
46+
* catalog
47+
*/
48+
private String catalog;
49+
4550
/**
4651
* database
4752
*/
@@ -84,6 +89,7 @@ public TransformContext(TransformContext context) {
8489
appendSemicolon = context.isAppendSemicolon();
8590
dataTypeTransformer = context.getDataTypeTransformer();
8691
this.viewSetting = context.getViewSetting();
92+
this.catalog = context.getCatalog();
8793
this.database = context.getDatabase();
8894
this.schema = context.getSchema();
8995
this.querySetting = context.getQuerySetting();
@@ -102,6 +108,7 @@ protected <T extends Builder<T>> TransformContext(Builder tBuilder) {
102108
dataTypeTransformer = tBuilder.dataTypeTransformer;
103109
viewSetting = tBuilder.viewSetting;
104110
this.querySetting = tBuilder.querySetting;
111+
this.catalog = tBuilder.catalog;
105112
this.database = tBuilder.database;
106113
this.schema = tBuilder.schema;
107114
this.prettyFormat = tBuilder.prettyFormat;
@@ -120,6 +127,11 @@ public static class Builder<T extends Builder<T>> {
120127

121128
private boolean appendSemicolon;
122129

130+
/**
131+
* catalog
132+
*/
133+
private String catalog;
134+
123135
/**
124136
* database
125137
*/
@@ -170,6 +182,11 @@ public T querySetting(QuerySetting querySetting) {
170182
return (T)this;
171183
}
172184

185+
public T catalog(String catalog) {
186+
this.catalog = catalog;
187+
return (T)this;
188+
}
189+
173190
public T database(String database) {
174191
this.database = database;
175192
return (T)this;

‎fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/dialect/DialectName.java

+1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ public enum DialectName implements IDialectName {
133133
* ObOracle
134134
*/
135135
OB_ORACLE(Constants.OB_ORACLE);
136+
136137
@Getter
137138
private final String value;
138139

0 commit comments

Comments
 (0)