Skip to content

Commit eb34e43

Browse files
authored
fix: databases and tables query failed in tdenigine (#29)
* fix: databases and tables query failed in tdenigine * add greptime compitable * chore: sort the databases and tables order * add e2e testing for greptime * chore: support the sql is empty * support postgres data query --------- Co-authored-by: rick <[email protected]>
1 parent 26b276c commit eb34e43

File tree

6 files changed

+257
-74
lines changed

6 files changed

+257
-74
lines changed

e2e/compose.yaml

+48-17
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ services:
77
depends_on:
88
mysql:
99
condition: service_healthy
10+
greptimedb:
11+
condition: service_healthy
12+
tdengine:
13+
condition: service_healthy
14+
postgres:
15+
condition: service_healthy
1016
extension:
1117
condition: service_started
1218
volumes:
@@ -15,20 +21,19 @@ services:
1521
target: /var/data
1622
links:
1723
- mysql
24+
- greptimedb
25+
- tdengine
26+
- postgres
1827
extension:
1928
build:
2029
context: ..
2130
dockerfile: Dockerfile
2231
args:
2332
- "GO_BUILDER=ghcr.io/linuxsuren/library/golang:1.22"
2433
- "BASE_IMAGE=ghcr.io/linuxsuren/library/alpine:3.12"
25-
ports:
26-
- "7071:7071"
27-
depends_on:
28-
mysql:
29-
condition: service_healthy
30-
links:
31-
- mysql
34+
- GOPROXY=${GOPROXY}
35+
# ports:
36+
# - "7071:7071"
3237
mysql:
3338
image: ghcr.io/linuxsuren/library/mysql:8.2.0
3439
command: --default-authentication-plugin=mysql_native_password
@@ -38,18 +43,44 @@ services:
3843
healthcheck:
3944
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306"]
4045
interval: 3s
41-
timeout: 60s
42-
retries: 10
43-
ports:
44-
- "3306:3306"
46+
timeout: 180s
47+
retries: 60
48+
# ports:
49+
# - "3306:3306"
4550

4651
tdengine:
47-
image: tdengine/tdengine:3.3.5.8
52+
image: ghcr.io/linuxsuren/tdengine/tdengine:3.3.3.0
53+
healthcheck:
54+
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/tdengine/6041"]
55+
interval: 3s
56+
timeout: 180s
57+
retries: 30
58+
# ports:
59+
# - "6030:6030" # REST API port
60+
# - "6031:6031" # client port
61+
# - "6041:6041" # cluster port
62+
63+
greptimedb:
64+
image: ghcr.io/linuxsuren/greptime/greptimedb:v0.12.0
65+
command: standalone start --mysql-addr=0.0.0.0:4002
66+
healthcheck:
67+
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/greptimedb/4002"]
68+
interval: 10s
69+
timeout: 5s
70+
retries: 3
71+
# ports:
72+
# - "4002:4002"
73+
74+
postgres:
75+
image: ghcr.io/linuxsuren/library/postgres:16.0
4876
environment:
49-
TAOS_ROOT_PASSWORD: "root"
50-
ports:
51-
- "6030:6030" # REST API port
52-
- "6031:6031" # client port
53-
- "6041:6041" # cluster port
77+
POSTGRES_USER: root
78+
POSTGRES_PASSWORD: root
79+
POSTGRES_DB: atest
80+
healthcheck:
81+
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"]
82+
interval: 3s
83+
timeout: 30s
84+
retries: 10
5485
volumes:
5586
cache:

e2e/entrypoint.sh

+5
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,9 @@ cmd="atest run -p test-suite.yaml"
1212
echo "start to run testing: $cmd"
1313
kind=orm target=mysql:3306 driver=mysql $cmd
1414

15+
kind=orm target=mysql driver=mysql atest run -p testing-data-query.yaml
16+
kind=orm target=greptimedb:4002 driver=greptime dbname=public atest run -p testing-data-query.yaml
17+
kind=orm target=tdengine:6041 driver=tdengine password=taosdata dbname=information_schema atest run -p testing-data-query.yaml
18+
kind=orm target=postgres driver=postgres atest run -p testing-data-query.yaml
19+
1520
cat /root/.config/atest/stores.yaml

e2e/testing-data-query.yaml

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!api-testing
2+
# yaml-language-server: $schema=https://linuxsuren.github.io/api-testing/api-testing-schema.json
3+
name: atest
4+
api: |
5+
{{default "http://localhost:8080" (env "SERVER")}}/api/v1
6+
param:
7+
store: "{{randAlpha 3}}"
8+
server: |
9+
{{default "http://localhost:8080" (env "SERVER")}}
10+
items:
11+
- name: CreateStore
12+
before:
13+
items:
14+
- httpReady("{{.param.server}}/healthz", 2400)
15+
request:
16+
api: /stores
17+
method: POST
18+
body: |
19+
{
20+
"name": "{{.param.store}}",
21+
"url": "{{env "target"}}",
22+
"username": "{{default "root" (env "username")}}",
23+
"password": "{{default "root" (env "password")}}",
24+
"kind": {
25+
"name": "atest-store-{{env "kind"}}"
26+
},
27+
"properties": [{
28+
"key": "driver",
29+
"value": "{{default "mysql" (env "driver")}}"
30+
}, {
31+
"key": "database",
32+
"value": "{{default "atest" (env "dbname")}}"
33+
}, {
34+
"key": "bucket",
35+
"value": "bucket"
36+
}, {
37+
"key": "region",
38+
"value": "cn"
39+
}, {
40+
"key": "disablessl",
41+
"value": "true"
42+
}, {
43+
"key": "targetPath",
44+
"value": "api-testing"
45+
}]
46+
}
47+
- name: query
48+
before:
49+
items:
50+
- sleep(3)
51+
request:
52+
api: /data/query
53+
method: POST
54+
header:
55+
X-Store-Name: "{{.param.store}}"
56+
body: |
57+
{
58+
"sql": "",
59+
"key": ""
60+
}

pkg/data_query.go

+108-41
Original file line numberDiff line numberDiff line change
@@ -21,69 +21,48 @@ import (
2121
"fmt"
2222
"github.com/linuxsuren/api-testing/pkg/server"
2323
"gorm.io/gorm"
24+
"log"
2425
"reflect"
26+
"sort"
27+
"strings"
2528
"time"
2629
)
2730

2831
func (s *dbserver) Query(ctx context.Context, query *server.DataQuery) (result *server.DataQueryResult, err error) {
2932
var db *gorm.DB
30-
if db, err = s.getClientWithDatabase(ctx, query.Key); err != nil {
33+
var dbQuery DataQuery
34+
if dbQuery, err = s.getClientWithDatabase(ctx, query.Key); err != nil {
3135
return
3236
}
3337

38+
db = dbQuery.GetClient()
39+
3440
result = &server.DataQueryResult{
3541
Data: []*server.Pair{},
3642
Items: make([]*server.Pairs, 0),
3743
Meta: &server.DataMeta{},
3844
}
3945

4046
// query database and tables
41-
queryDatabaseSql := "show databases"
42-
var databaseResult *server.DataQueryResult
43-
if databaseResult, err = sqlQuery(ctx, queryDatabaseSql, db); err == nil {
44-
for _, table := range databaseResult.Items {
45-
for _, item := range table.GetData() {
46-
if item.Key == "Database" {
47-
var found bool
48-
for _, name := range result.Meta.Databases {
49-
if name == item.Value {
50-
found = true
51-
}
52-
}
53-
if !found {
54-
result.Meta.Databases = append(result.Meta.Databases, item.Value)
55-
}
56-
}
57-
}
58-
}
47+
if result.Meta.Databases, err = dbQuery.GetDatabases(ctx); err != nil {
48+
log.Printf("failed to query databases: %v\n", err)
5949
}
6050

61-
var row *sql.Row
62-
if row = db.Raw("SELECT DATABASE() as name").Row(); row != nil {
63-
if err = row.Scan(&result.Meta.CurrentDatabase); err == nil {
64-
queryTableSql := "show tables"
65-
var tableResult *server.DataQueryResult
66-
if tableResult, err = sqlQuery(ctx, queryTableSql, db); err == nil {
67-
for _, table := range tableResult.Items {
68-
for _, item := range table.GetData() {
69-
if item.Key == fmt.Sprintf("Tables_in_%s", result.Meta.CurrentDatabase) {
70-
var found bool
71-
for _, name := range result.Meta.Tables {
72-
if name == item.Value {
73-
found = true
74-
}
75-
}
76-
if !found {
77-
result.Meta.Tables = append(result.Meta.Tables, item.Value)
78-
}
79-
}
80-
}
81-
}
82-
}
51+
if result.Meta.CurrentDatabase = query.Key; query.Key == "" {
52+
if result.Meta.CurrentDatabase, err = dbQuery.GetCurrentDatabase(); err != nil {
53+
log.Printf("failed to query current database: %v\n", err)
8354
}
8455
}
8556

57+
if result.Meta.Tables, err = dbQuery.GetTables(ctx, result.Meta.CurrentDatabase); err != nil {
58+
log.Printf("failed to query tables: %v\n", err)
59+
}
60+
8661
// query data
62+
if query.Sql == "" {
63+
return
64+
}
65+
8766
var dataResult *server.DataQueryResult
8867
if dataResult, err = sqlQuery(ctx, query.Sql, db); err == nil {
8968
result.Items = dataResult.Items
@@ -177,3 +156,91 @@ func sqlQuery(ctx context.Context, sql string, db *gorm.DB) (result *server.Data
177156
}
178157
return
179158
}
159+
160+
const queryDatabaseSql = "show databases"
161+
162+
type DataQuery interface {
163+
GetDatabases(context.Context) (databases []string, err error)
164+
GetTables(ctx context.Context, currentDatabase string) (tables []string, err error)
165+
GetCurrentDatabase() (string, error)
166+
GetClient() *gorm.DB
167+
}
168+
169+
type commonDataQuery struct {
170+
showDatabases, showTables, currentDatabase string
171+
db *gorm.DB
172+
}
173+
174+
var _ DataQuery = &commonDataQuery{}
175+
176+
func NewCommonDataQuery(showDatabases, showTables, currentDatabase string, db *gorm.DB) DataQuery {
177+
return &commonDataQuery{
178+
showDatabases: showDatabases,
179+
showTables: showTables,
180+
currentDatabase: currentDatabase,
181+
db: db,
182+
}
183+
}
184+
185+
func (q *commonDataQuery) GetDatabases(ctx context.Context) (databases []string, err error) {
186+
var databaseResult *server.DataQueryResult
187+
if databaseResult, err = sqlQuery(ctx, q.showDatabases, q.db); err == nil {
188+
for _, table := range databaseResult.Items {
189+
for _, item := range table.GetData() {
190+
if item.Key == "Database" || item.Key == "name" {
191+
var found bool
192+
for _, name := range databases {
193+
if name == item.Value {
194+
found = true
195+
}
196+
}
197+
if !found {
198+
databases = append(databases, item.Value)
199+
}
200+
}
201+
}
202+
}
203+
sort.Strings(databases)
204+
}
205+
return
206+
}
207+
208+
func (q *commonDataQuery) GetTables(ctx context.Context, currentDatabase string) (tables []string, err error) {
209+
showTables := q.showTables
210+
if strings.Contains(showTables, "%s") {
211+
showTables = fmt.Sprintf(showTables, currentDatabase)
212+
}
213+
214+
var tableResult *server.DataQueryResult
215+
if tableResult, err = sqlQuery(ctx, showTables, q.db); err == nil {
216+
for _, table := range tableResult.Items {
217+
for _, item := range table.GetData() {
218+
if item.Key == fmt.Sprintf("Tables_in_%s", currentDatabase) || item.Key == "table_name" ||
219+
item.Key == "Tables" || item.Key == "tablename" {
220+
var found bool
221+
for _, name := range tables {
222+
if name == item.Value {
223+
found = true
224+
}
225+
}
226+
if !found {
227+
tables = append(tables, item.Value)
228+
}
229+
}
230+
}
231+
}
232+
sort.Strings(tables)
233+
}
234+
return
235+
}
236+
func (q *commonDataQuery) GetCurrentDatabase() (current string, err error) {
237+
var row *sql.Row
238+
if row = q.db.Raw(q.currentDatabase).Row(); row != nil {
239+
err = row.Scan(&current)
240+
}
241+
return
242+
}
243+
244+
func (q *commonDataQuery) GetClient() *gorm.DB {
245+
return q.db
246+
}

0 commit comments

Comments
 (0)