Skip to content

Commit bccdc31

Browse files
YANG-DBYANG-DB
authored
update cloud trail integration with flint-s3 based DDL assets and das… (#1701)
* update cloud trail integration with flint-s3 based DDL assets and dashboard Signed-off-by: YANGDB <[email protected]> * update time field using CAST to TIMESTAMP Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update refresh sync rate Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]>
1 parent 3506f93 commit bccdc31

File tree

4 files changed

+181
-2
lines changed

4 files changed

+181
-2
lines changed

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/aws_cloudtrail-flint-1.0.0.ndjson

+18
Large diffs are not rendered by default.

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_mv_cloud-trail-1.0.0.sql

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
CREATE MATERIALIZED VIEW {table_name}_mview AS
2+
SELECT
3+
rec.userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
4+
rec.userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
5+
rec.userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
6+
rec.userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
7+
rec.userIdentity.invokedBy AS `aws.cloudtrail.userIdentity.invokedBy`,
8+
rec.userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
9+
rec.userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
10+
rec.userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
11+
CAST(rec.userIdentity.sessionContext.attributes.creationDate AS TIMESTAMP) AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
12+
rec.userIdentity.sessionContext.sessionIssuer.type AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type`,
13+
rec.userIdentity.sessionContext.sessionIssuer.principalId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.principalId`,
14+
rec.userIdentity.sessionContext.sessionIssuer.arn AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.arn`,
15+
rec.userIdentity.sessionContext.sessionIssuer.accountId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.accountId`,
16+
rec.userIdentity.sessionContext.sessionIssuer.userName AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.userName`,
17+
rec.userIdentity.sessionContext.ec2RoleDelivery AS `aws.cloudtrail.userIdentity.sessionContext.ec2RoleDelivery`,
18+
19+
rec.eventVersion AS `aws.cloudtrail.eventVersion`,
20+
CAST(rec.eventTime AS TIMESTAMP) AS `@timestamp`,
21+
rec.eventSource AS `aws.cloudtrail.eventSource`,
22+
rec.eventName AS `aws.cloudtrail.eventName`,
23+
rec.eventCategory AS `aws.cloudtrail.eventCategory`,
24+
rec.eventType AS `aws.cloudtrail.eventType`,
25+
rec.eventId AS `aws.cloudtrail.eventId`,
26+
27+
rec.awsRegion AS `aws.cloudtrail.awsRegion`,
28+
rec.sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
29+
rec.userAgent AS `aws.cloudtrail.userAgent`,
30+
rec.errorCode AS `errorCode`,
31+
rec.errorMessage AS `errorMessage`,
32+
rec.requestParameters AS `aws.cloudtrail.requestParameter`,
33+
rec.responseElements AS `aws.cloudtrail.responseElements`,
34+
rec.additionalEventData AS `aws.cloudtrail.additionalEventData`,
35+
rec.requestId AS `aws.cloudtrail.requestId`,
36+
rec.resources AS `aws.cloudtrail.resources`,
37+
rec.apiVersion AS `aws.cloudtrail.apiVersion`,
38+
rec.readOnly AS `aws.cloudtrail.readOnly`,
39+
rec.recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
40+
rec.serviceEventDetails AS `aws.cloudtrail.serviceEventDetails`,
41+
rec.sharedEventId AS `aws.cloudtrail.sharedEventId`,
42+
rec.vpcEndpointId AS `aws.cloudtrail.vpcEndpointId`,
43+
rec.tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tls_version`,
44+
rec.tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetailscipher_suite`,
45+
rec.tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetailsclient_provided_host_header`
46+
FROM
47+
{table_name}
48+
LATERAL VIEW explode(Records) myTable AS rec
49+
WITH (
50+
auto_refresh = true,
51+
checkpoint_location = '{s3_checkpoint_location}',
52+
watermark_delay = '1 min',
53+
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
54+
)

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
2+
Records ARRAY<STRUCT<
3+
eventVersion STRING,
4+
userIdentity STRUCT<
5+
type:STRING,
6+
principalId:STRING,
7+
arn:STRING,
8+
accountId:STRING,
9+
invokedBy:STRING,
10+
accessKeyId:STRING,
11+
userName:STRING,
12+
sessionContext:STRUCT<
13+
attributes:STRUCT<
14+
mfaAuthenticated:STRING,
15+
creationDate:STRING
16+
>,
17+
sessionIssuer:STRUCT<
18+
type:STRING,
19+
principalId:STRING,
20+
arn:STRING,
21+
accountId:STRING,
22+
userName:STRING
23+
>,
24+
ec2RoleDelivery:STRING,
25+
webIdFederationData:MAP<STRING,STRING>
26+
>
27+
>,
28+
eventTime STRING,
29+
eventSource STRING,
30+
eventName STRING,
31+
awsRegion STRING,
32+
sourceIPAddress STRING,
33+
userAgent STRING,
34+
errorCode STRING,
35+
errorMessage STRING,
36+
requestParameters STRING,
37+
responseElements STRING,
38+
additionalEventData STRING,
39+
requestId STRING,
40+
eventId STRING,
41+
resources ARRAY<STRUCT<
42+
arn:STRING,
43+
accountId:STRING,
44+
type:STRING
45+
>>,
46+
eventType STRING,
47+
apiVersion STRING,
48+
readOnly STRING,
49+
recipientAccountId STRING,
50+
serviceEventDetails STRING,
51+
sharedEventId STRING,
52+
vpcEndpointId STRING,
53+
eventCategory STRING,
54+
tlsDetails STRUCT<
55+
tlsVersion:STRING,
56+
cipherSuite:STRING,
57+
clientProvidedHostHeader:STRING
58+
>
59+
>>
60+
) USING json
61+
LOCATION '{s3_bucket_location}'
62+
OPTIONS (
63+
compression='gzip',
64+
recursivefilelookup='true'
65+
);

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json

+44-2
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,29 @@
55
"description": "Analyze CloudTrail events.",
66
"license": "Apache-2.0",
77
"type": "logs-aws_cloudtrail",
8-
"labels": ["Observability", "Logs", "AWS", "Cloud"],
8+
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "OpenSearch",
1010
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
11+
"workflows": [
12+
{
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
17+
},
18+
{
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
23+
},
24+
{
25+
"name": "dashboards-flint",
26+
"label": "Dashboards & Visualizations adapted to Flint",
27+
"description": "Dashboards and visualizations adapted to Flint S3 datasource that enable you to easily visualize data residing on S3",
28+
"enabled_by_default": false
29+
}
30+
],
1131
"statics": {
1232
"logo": {
1333
"annotation": "CloudTrail Logo",
@@ -43,7 +63,29 @@
4363
"name": "aws_cloudtrail",
4464
"version": "1.0.0",
4565
"extension": "ndjson",
46-
"type": "savedObjectBundle"
66+
"type": "savedObjectBundle",
67+
"workflows": ["dashboards"]
68+
},
69+
{
70+
"name": "aws_cloudtrail-flint",
71+
"version": "1.0.0",
72+
"extension": "ndjson",
73+
"type": "savedObjectBundle",
74+
"workflows": ["dashboards-flint"]
75+
},
76+
{
77+
"name": "create_table_cloud-trail",
78+
"version": "1.0.0",
79+
"extension": "sql",
80+
"type": "query",
81+
"workflows": ["dashboards-flint"]
82+
},
83+
{
84+
"name": "create_mv_cloud-trail",
85+
"version": "1.0.0",
86+
"extension": "sql",
87+
"type": "query",
88+
"workflows": ["dashboards-flint"]
4789
}
4890
],
4991
"sampleData": {

0 commit comments

Comments
 (0)