Skip to content

Commit 947385f

Browse files
YANG-DBYANG-DB
authored
add saved queries to vpc flow (#1744)
* add saved queries to vpc flow Signed-off-by: YANGDB <[email protected]> * add skipping index creation Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]>
1 parent 3dbe5c8 commit 947385f

File tree

3 files changed

+41
-9
lines changed

3 files changed

+41
-9
lines changed

‎server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
accountid BLOOM_FILTER,
3+
region VALUE_SET,
4+
severity_id VALUE_SET,
5+
src_endpoint.ip BLOOM_FILTER,
6+
dst_endpoint.ip BLOOM_FILTER,
7+
src_endpoint.svc_name VALUE_SET,
8+
dst_endpoint.svc_name VALUE_SET,
9+
request_processing_time MIN_MAX,
10+
traffic.bytes MIN_MAX
11+
) WITH (
12+
auto_refresh = true,
13+
refresh_interval = '15 Minutes',
14+
checkpoint_location = '{s3_checkpoint_location}',
15+
watermark_delay = '1 Minute'
16+
)

‎server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"attributes":{"createdTimeMs":1713289099101,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"agg_60_min_connections_view","query":"SELECT date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time, date_trunc('hour', from_unixtime(start_time / 1000)) + INTERVAL 1 HOUR AS interval_end_time, accountid as `aws.vpc.account-id`, region as `aws.vpc.region`, COUNT(*) AS total_connections, SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes, SUM(CAST(IFNULL(traffic.packets, 0) AS LONG)) AS total_packets FROM {table_name} GROUP BY date_trunc('hour', from_unixtime(start_time / 1000)), region, accountid\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Hourly count connections summary","version":1},"id":"1d07d010-fc18-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:52:30.414Z","version":"WzI3NTEsMV0="}
2+
{"attributes":{"createdTimeMs":1713293044079,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"window_agg_60_min_network_ip_bytes","query":"WITH hourly_buckets AS ( SELECT date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time, CAST(IFNULL(dst_endpoint.ip, '0.0.0.0') AS STRING) AS dstaddr, SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes FROM {table_name} GROUP BY interval_start_time, dstaddr ), ranked_addresses AS ( SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes, RANK() OVER (PARTITION BY interval_start_time ORDER BY total_bytes DESC) AS bytes_rank FROM hourly_buckets ) SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes FROM ranked_addresses WHERE bytes_rank <= 50 ORDER BY interval_start_time ASC, bytes_rank ASC","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"window hourly network ip bytes summary","version":1},"id":"4c6b8820-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:44:47.956Z","version":"WzI4MzAsMV0="}
3+
{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"TopCommonErrorServicesQuery","query":"SELECT src_endpoint.svc_name AS source_service, dst_endpoint.svc_name AS destination_service, COUNT(*) AS error_count FROM {table_name} WHERE severity_id >= 4 GROUP BY src_endpoint.svc_name, dst_endpoint.svc_name ORDER BY error_count DESC LIMIT 10;\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Top 10 pairs of errored source / destination services","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
4+
{"exportedCount":7,"missingRefCount":0,"missingReferences":[]}

‎server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,27 @@
7575
}
7676
],
7777
"assets": [
78+
{
79+
"name": "create_table_parquet_vpc",
80+
"version": "1.0.0",
81+
"extension": "sql",
82+
"type": "query",
83+
"workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"]
84+
},
85+
{
86+
"name": "example_queries",
87+
"version": "1.0.0",
88+
"extension": "ndjson",
89+
"type": "savedObjectBundle",
90+
"workflows": ["queries"]
91+
},
92+
{
93+
"name": "create_skipping_index",
94+
"version": "1.0.0",
95+
"extension": "sql",
96+
"type": "query",
97+
"workflows": ["queries"]
98+
},
7899
{
79100
"name": "aws_vpc_flow",
80101
"version": "1.0.0",
@@ -96,14 +117,6 @@
96117
"type": "savedObjectBundle",
97118
"workflows": ["flint-pre-agg-dashboards"]
98119
},
99-
100-
{
101-
"name": "create_table_parquet_vpc",
102-
"version": "1.0.0",
103-
"extension": "sql",
104-
"type": "query",
105-
"workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"]
106-
},
107120
{
108121
"name": "vpc_live_all_mv",
109122
"version": "1.0.0",
@@ -139,7 +152,6 @@
139152
"type": "query",
140153
"workflows": ["flint-pre-agg-dashboards"]
141154
},
142-
143155
{
144156
"name": "vpc_live_week_refresh",
145157
"version": "1.0.0",

0 commit comments

Comments
 (0)