Skip to content

Commit 1944d0b

Browse files
RyanL1997RyanL1997
authored
[Integration] Integration enhancements (#1860)
* Add recursive file lookup for all table creation queries Signed-off-by: Ryan Liang <[email protected]> * Change the versions and source urls Signed-off-by: Ryan Liang <[email protected]> * Change the source url to release of catalog repo, and fix the formats for apache and nginx Signed-off-by: Ryan Liang <[email protected]> * Reverted some version changes on integrations with only doc link changes Signed-off-by: Ryan Liang <[email protected]> --------- Signed-off-by: Ryan Liang <[email protected]>
1 parent c7f7f95 commit 1944d0b

File tree

16 files changed

+115
-101
lines changed

16 files changed

+115
-101
lines changed

‎server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/amazon_cloudfront/amazon_cloudfront-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "amazon_cloudfront",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "Amazon CloudFront",
55
"description": "Analyze access logs for Amazon CloudFront.",
66
"license": "Apache-2.0",
77
"type": "logs-aws_cloudfront",
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_cloudfront-1.0.0",
1111
"statics": {
1212
"logo": {
1313
"annotation": "CloudFront Logo",

‎server/adaptors/integrations/__data__/repository/amazon_cloudfront/assets/create_table-1.0.0.sql

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
3535
)
3636
USING csv
3737
LOCATION '{s3_bucket_location}'
38-
OPTIONS ( sep='\t' )
38+
OPTIONS (
39+
sep='\t',
40+
recursiveFileLookup='true'
41+
)
3942
TBLPROPERTIES ( 'skip.header.line.count'='2' )

‎server/adaptors/integrations/__data__/repository/amazon_elb/amazon_elb-1.0.0.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"type": "logs_elb",
88
"labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_elb-1.0.0",
1111
"workflows": [
1212
{
1313
"name": "queries",

‎server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/amazon_s3/amazon_s3-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "amazon_s3",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "Amazon S3",
55
"description": "Analyze Amazon S3 server logs.",
66
"license": "Apache-2.0",
77
"type": "logs_s3",
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_s3_access-logs-1.1.0",
1111
"workflows": [
1212
{
1313
"name": "queries",

‎server/adaptors/integrations/__data__/repository/amazon_s3/assets/create_table-1.0.0.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
2727
)
2828
USING csv
2929
OPTIONS (
30-
sep=' '
30+
sep=' ',
31+
recursiveFileLookup='true'
3132
)
3233
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/amazon_vpc_flow-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "amazon_vpc_flow",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "Amazon VPC",
55
"description": "Monitor IP traffic with VPC Flow Logs.",
66
"license": "Apache-2.0",
77
"type": "logs_vpc",
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/amazon_vpc_flow_1.1.0",
1111
"workflows": [
1212
{
1313
"name": "queries",

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_vpc_schema-1.0.0.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
2828
pkt_dst_aws_service string,
2929
flow_direction string,
3030
traffic_path int
31-
) USING parquet
31+
)
32+
USING parquet
3233
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json

+37-37
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,52 @@
11
{
2-
name: "apache",
3-
version: "1.0.0",
4-
displayName: "Apache Access Logs",
5-
description: "Insights on applications using the standard Apache access log format.",
6-
license: "Apache-2.0",
7-
type: "logs_apache",
8-
labels: ["Observability", "Logs", "Flint S3"],
9-
author: "OpenSearch",
10-
sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
11-
workflows: [
2+
"name": "apache",
3+
"version": "1.0.0",
4+
"displayName": "Apache Access Logs",
5+
"description": "Insights on applications using the standard Apache access log format.",
6+
"license": "Apache-2.0",
7+
"type": "logs_apache",
8+
"labels": ["Observability", "Logs", "Flint S3"],
9+
"author": "OpenSearch",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/apache-1.0.0",
11+
"workflows": [
1212
{
13-
name: "queries",
14-
label: "Queries (recommended)",
15-
description: "Tables and pre-written queries for quickly getting insights on your data.",
16-
enabled_by_default: true
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
1717
},
1818
{
19-
name: "dashboards",
20-
label: "Dashboards & Visualizations",
21-
description: "Dashboards and indices that enable you to easily visualize important metrics.",
22-
enabled_by_default: false
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
2323
}
2424
],
25-
statics: {
26-
logo: {
27-
annotation: "Apache Logo",
28-
path: "logo.png"
25+
"statics": {
26+
"logo": {
27+
"annotation": "Apache Logo",
28+
"path": "logo.png"
2929
},
30-
gallery: [
30+
"gallery": [
3131
{
32-
annotation: "Apache Dashboard",
33-
path: "dashboard1.png"
32+
"annotation": "Apache Dashboard",
33+
"path": "dashboard1.png"
3434
}
3535
]
3636
},
37-
components: [
38-
{ name: "communication", version: "1.0.0" },
39-
{ name: "http", version: "1.0.0" },
40-
{ name: "logs_apache", version: "1.0.0" }
37+
"components": [
38+
{ "name": "communication", "version": "1.0.0" },
39+
{ "name": "http", "version": "1.0.0" },
40+
{ "name": "logs_apache", "version": "1.0.0" }
4141
],
42-
assets: [
43-
{ name: "apache", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
44-
{ name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] },
45-
{ name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
46-
{ name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
47-
{ name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] }
42+
"assets": [
43+
{ "name": "apache", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
44+
{ "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] },
45+
{ "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
46+
{ "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
47+
{ "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] }
4848
],
49-
sampleData: {
50-
path: "sample.json"
49+
"sampleData": {
50+
"path": "sample.json"
5151
}
5252
}

‎server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql

+5-3
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
99
body_bytes_sent INT,
1010
http_referer STRING,
1111
http_user_agent STRING
12-
) USING csv
13-
LOCATION '{s3_bucket_location}'
12+
)
13+
USING csv
1414
OPTIONS (
1515
sep=' ',
16-
nullValue='-'
16+
nullValue='-',
17+
recursiveFileLookup='true'
1718
)
19+
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "aws_cloudtrail",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "AWS CloudTrail",
55
"description": "Analyze CloudTrail events.",
66
"license": "Apache-2.0",
77
"type": "logs-aws_cloudtrail",
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_cloudtrail-1.1.0",
1111
"workflows": [
1212
{
1313
"name": "dashboards",

‎server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql

+3
Original file line numberDiff line numberDiff line change
@@ -125,4 +125,7 @@ CREATE EXTERNAL TABLE {table_name} (
125125
`ja3Fingerprint` string
126126
)
127127
USING json
128+
OPTIONS (
129+
recursiveFileLookup='true'
130+
)
128131
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "aws_waf",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "AWS Web Application Firewall",
55
"description": "Analyze AWS WAF web ACL traffic.",
66
"license": "Apache-2.0",
77
"type": "logs_waf",
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/aws_waf-1.0.0",
1111
"workflows": [
1212
{
1313
"name": "queries",

‎server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
CREATE EXTERNAL TABLE {table_name} (
22
record STRING
3-
) USING csv
4-
LOCATION '{s3_bucket_location}'
3+
)
4+
USING csv
55
OPTIONS (
6-
sep='\0x1E'
6+
sep='\0x1E',
7+
recursiveFileLookup='true'
78
)
9+
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json renamed to ‎server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.1.0.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
{
22
"name": "haproxy",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"displayName": "HAProxy",
55
"description": "Analyze HAProxy access logs.",
66
"license": "Apache-2.0",
77
"type": "logs",
88
"labels": ["Observability", "Logs", "Flint S3"],
99
"author": "OpenSearch",
10-
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/haproxy-1.0.0",
1111
"workflows": [
1212
{
1313
"name": "queries",

‎server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql

+5-3
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@ CREATE EXTERNAL TABLE {table_name} (
1010
http_referer STRING,
1111
http_user_agent STRING,
1212
gzip_ratio STRING
13-
) USING csv
14-
LOCATION '{s3_bucket_location}'
13+
)
14+
USING csv
1515
OPTIONS (
1616
sep=' ',
17-
nullValue='-'
17+
nullValue='-',
18+
recursiveFileLookup='true'
1819
)
20+
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

+39-39
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,56 @@
11
{
2-
name: "nginx",
3-
version: "1.0.0",
4-
displayName: "Nginx",
5-
description: "Analyze Nginx access logs.",
6-
license: "Apache-2.0",
7-
type: "logs",
8-
labels: ["Observability", "Logs", "Flint S3"],
9-
author: "OpenSearch",
10-
sourceUrl: "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
11-
workflows: [
2+
"name": "nginx",
3+
"version": "1.0.0",
4+
"displayName": "Nginx",
5+
"description": "Analyze Nginx access logs.",
6+
"license": "Apache-2.0",
7+
"type": "logs",
8+
"labels": ["Observability", "Logs", "Flint S3"],
9+
"author": "OpenSearch",
10+
"sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/releases/tag/nginx-1.0.0",
11+
"workflows": [
1212
{
13-
name: "queries",
14-
label: "Queries (recommended)",
15-
description: "Tables and pre-written queries for quickly getting insights on your data.",
16-
enabled_by_default: true
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
1717
},
1818
{
19-
name: "dashboards",
20-
label: "Dashboards & Visualizations",
21-
description: "Dashboards and indices that enable you to easily visualize important metrics.",
22-
enabled_by_default: false
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
2323
}
2424
],
25-
statics: {
26-
logo: {
27-
annotation: "NginX Logo",
28-
path: "logo.svg"
25+
"statics": {
26+
"logo": {
27+
"annotation": "NginX Logo",
28+
"path": "logo.svg"
2929
},
30-
gallery: [
30+
"gallery": [
3131
{
32-
annotation: "NginX Dashboard",
33-
path: "dashboard1.png"
32+
"annotation": "NginX Dashboard",
33+
"path": "dashboard1.png"
3434
},
3535
{
36-
annotation: "NginX Dashboard view",
37-
path: "dashboard2.png"
36+
"annotation": "NginX Dashboard view",
37+
"path": "dashboard2.png"
3838
}
3939
]
4040
},
41-
components: [
42-
{ name: "communication", version: "1.0.0" },
43-
{ name: "http", version: "1.0.0" },
44-
{ name: "logs", version: "1.0.0" }
41+
"components": [
42+
{ "name": "communication", "version": "1.0.0" },
43+
{ "name": "http", "version": "1.0.0" },
44+
{ "name": "logs", "version": "1.0.0" }
4545
],
46-
assets: [
47-
{ name: "nginx", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["dashboards"] },
48-
{ name: "create_table", version: "1.0.0", extension: "sql", type: "query" },
49-
{ name: "create_skipping_index", version: "1.0.0", extension: "sql", type: "query", workflows: ["queries"] },
50-
{ name: "create_mv", version: "1.0.0", extension: "sql", type: "query", workflows: ["dashboards"] },
51-
{ name: "example_queries", version: "1.0.0", extension: "ndjson", type: "savedObjectBundle", workflows: ["queries"] }
46+
"assets": [
47+
{ "name": "nginx", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["dashboards"] },
48+
{ "name": "create_table", "version": "1.0.0", "extension": "sql", "type": "query" },
49+
{ "name": "create_skipping_index", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["queries"] },
50+
{ "name": "create_mv", "version": "1.0.0", "extension": "sql", "type": "query", "workflows": ["dashboards"] },
51+
{ "name": "example_queries", "version": "1.0.0", "extension": "ndjson", "type": "savedObjectBundle", "workflows": ["queries"] }
5252
],
53-
sampleData: {
54-
path: "sample.json"
53+
"sampleData": {
54+
"path": "sample.json"
5555
}
5656
}

0 commit comments

Comments
 (0)