Skip to content

Commit 685aeaf

Browse files
SwiddisSwiddis
authored
Add skipping indices for all integrations that have sample queries (#1747)
* Update avg threshold for cloudfront query Signed-off-by: Simeon Widdis <[email protected]> * Add first draft skipping index for cloudfront integration Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for nginx Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for s3 Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for elb Signed-off-by: Simeon Widdis <[email protected]> * Update skipping indices to use bloom filters for client IDs Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for apache Signed-off-by: Simeon Widdis <[email protected]> --------- Signed-off-by: Simeon Widdis <[email protected]>
1 parent 947385f commit 685aeaf

11 files changed

+95
-9
lines changed

‎server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@
6969
"extension": "sql",
7070
"type": "query"
7171
},
72+
{
73+
"name": "create_skipping_index",
74+
"version": "1.0.0",
75+
"extension": "sql",
76+
"type": "query",
77+
"workflows": ["queries"]
78+
},
7279
{
7380
"name": "create_mv",
7481
"version": "1.0.0",

‎server/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
remote_addr BLOOM_FILTER,
3+
`status` VALUE_SET,
4+
body_bytes_sent MIN_MAX
5+
) WITH (
6+
auto_refresh = true,
7+
refresh_interval = '15 Minutes',
8+
checkpoint_location = '{s3_checkpoint_location}',
9+
watermark_delay = '1 Minute'
10+
)

‎server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
c_ip BLOOM_FILTER,
3+
sc_status VALUE_SET,
4+
time_to_first_byte MIN_MAX
5+
) WITH (
6+
auto_refresh = true,
7+
refresh_interval = '15 Minutes',
8+
checkpoint_location = '{s3_checkpoint_location}',
9+
watermark_delay = '1 Minute'
10+
)

‎server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@
33
{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Number of Requests","query":"SELECT COUNT(*) AS request_count FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Number of Requests","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
44
{"attributes":{"createdTimeMs":1713293161193,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Total Bytes Served","query":"SELECT SUM(sc_bytes) AS total_bytes_served FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Total Bytes Served","version":1},"id":"92398eb0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:46:01.242Z","version":"WzI4MzEsMV0="}
55
{"attributes":{"createdTimeMs":1713293269224,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Average Time Taken","query":"SELECT AVG(time_taken) AS average_time_taken FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Average Time Taken","version":1},"id":"d2a038a0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:47:49.290Z","version":"WzI4MzIsMV0="}
6-
{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 1 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="}
6+
{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 2 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="}
77
{"attributes":{"createdTimeMs":1713294061574,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Requests by User Agent","query":"SELECT * FROM {table_name} WHERE cs_user_agent LIKE '%Chrome%' LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Requests by User Agent","version":1},"id":"aae73c80-fc23-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T19:01:01.640Z","version":"WzI4MzUsMV0="}
88
{"exportedCount":7,"missingRefCount":0,"missingReferences":[]}

‎server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,17 +54,24 @@
5454
],
5555
"assets": [
5656
{
57-
"name": "aws_cloudfront",
57+
"name": "create_table",
5858
"version": "1.0.0",
59-
"extension": "ndjson",
60-
"type": "savedObjectBundle",
61-
"workflows": ["dashboards"]
59+
"extension": "sql",
60+
"type": "query"
6261
},
6362
{
64-
"name": "create_table",
63+
"name": "create_skipping_index",
6564
"version": "1.0.0",
6665
"extension": "sql",
67-
"type": "query"
66+
"type": "query",
67+
"workflows": ["queries"]
68+
},
69+
{
70+
"name": "example_queries",
71+
"version": "1.0.0",
72+
"extension": "ndjson",
73+
"type": "savedObjectBundle",
74+
"workflows": ["queries"]
6875
},
6976
{
7077
"name": "create_mv",
@@ -74,11 +81,11 @@
7481
"workflows": ["dashboards"]
7582
},
7683
{
77-
"name": "example_queries",
84+
"name": "aws_cloudfront",
7885
"version": "1.0.0",
7986
"extension": "ndjson",
8087
"type": "savedObjectBundle",
81-
"workflows": ["queries"]
88+
"workflows": ["dashboards"]
8289
}
8390
],
8491
"sampleData": {

‎server/adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
client_ip BLOOM_FILTER,
3+
elb_status_code VALUE_SET,
4+
request_processing_time MIN_MAX,
5+
sent_bytes MIN_MAX
6+
) WITH (
7+
auto_refresh = true,
8+
refresh_interval = '15 Minutes',
9+
checkpoint_location = '{s3_checkpoint_location}',
10+
watermark_delay = '1 Minute'
11+
)

‎server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@
7474
"extension": "sql",
7575
"type": "query"
7676
},
77+
{
78+
"name": "create_skipping_index",
79+
"version": "1.0.0",
80+
"extension": "sql",
81+
"type": "query",
82+
"workflows": ["queries"]
83+
},
7784
{
7885
"name": "create_mv",
7986
"version": "1.0.0",

‎server/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
requester BLOOM_FILTER,
3+
http_status VALUE_SET,
4+
request_time MIN_MAX
5+
) WITH (
6+
auto_refresh = true,
7+
refresh_interval = '15 Minutes',
8+
checkpoint_location = '{s3_checkpoint_location}',
9+
watermark_delay = '1 Minute'
10+
)

‎server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@
6969
"extension": "sql",
7070
"type": "query"
7171
},
72+
{
73+
"name": "create_skipping_index",
74+
"version": "1.0.0",
75+
"extension": "sql",
76+
"type": "query",
77+
"workflows": ["queries"]
78+
},
7279
{
7380
"name": "create_mv",
7481
"version": "1.0.0",

‎server/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE SKIPPING INDEX ON {table_name} (
2+
remote_addr BLOOM_FILTER,
3+
`status` VALUE_SET,
4+
body_bytes_sent MIN_MAX
5+
) WITH (
6+
auto_refresh = true,
7+
refresh_interval = '15 Minutes',
8+
checkpoint_location = '{s3_checkpoint_location}',
9+
watermark_delay = '1 Minute'
10+
)

‎server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,13 @@
6666
"extension": "sql",
6767
"type": "query"
6868
},
69+
{
70+
"name": "create_skipping_index",
71+
"version": "1.0.0",
72+
"extension": "sql",
73+
"type": "query",
74+
"workflows": ["queries"]
75+
},
6976
{
7077
"name": "create_mv",
7178
"version": "1.0.0",

0 commit comments

Comments
 (0)