[Integrations] Add integration of S3 Access log (#1684)

* Add integration of S3 Access log

Signed-off-by: Ryan Liang <[email protected]>

* Correct the checkpoint placeholder

Signed-off-by: Ryan Liang <[email protected]>

* Correct the parsing of timestamp

Signed-off-by: Ryan Liang <[email protected]>

* Correct the auto refresh config boolean

Signed-off-by: Ryan Liang <[email protected]>

---------

Signed-off-by: Ryan Liang <[email protected]>
(cherry picked from commit ee0d748)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: github-actions[bot]

server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql

@@ -0,0 +1,35 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+SELECT
+  owner_id AS `aws.s3.bucket_owner`,
+  bucket_name AS `aws.s3.bucket`,
+  to_timestamp(CONCAT(SUBSTRING(request_time, 2), ' ', SUBSTRING(request_time_zone, 1, LENGTH(request_time_zone) - 1)), 'dd/MMM/yyyy:HH:mm:ss Z') AS `@timestamp`,
+  CONCAT(request_time, ' ', request_time_zone) AS `aws.s3.request_time`,
+  remote_ip AS `aws.s3.remote_ip`,
+  requester AS `aws.s3.requester`,
+  request_id AS `aws.s3.request_id`,
+  operation AS `aws.s3.operation`,
+  key AS `aws.s3.key`,
+  request_uri AS `aws.s3.request_uri`,
+  http_status AS `aws.s3.http_status`,
+  error_code AS `aws.s3.error_code`,
+  bytes_sent AS `aws.s3.bytes_sent`,
+  object_size AS `aws.s3.object_size`,
+  CAST(total_time AS INTEGER) AS `aws.s3.total_time`,
+  CAST(turn_around_time AS INTEGER) AS `aws.s3.turn_around_time`,
+  referrer AS `aws.s3.referrer`,
+  user_agent AS `aws.s3.user_agent`,
+  version_id AS `aws.s3.version_id`,
+  host_id AS `aws.s3.host_id`,
+  signature_version AS `aws.s3.signature_version`,
+  cipher_suite AS `aws.s3.cipher_suite`,
+  authentication_type AS `aws.s3.authentication_type`,
+  host_header AS `aws.s3.host_header`,
+  tls_version AS `aws.s3.tls_version`
+FROM
+  {table_name}
+WITH (
+  auto_refresh = true,
+  checkpoint_location = '{s3_checkpoint_location}',
+  watermark_delay = '1 Minute',
+  extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);

server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql

@@ -0,0 +1,32 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+  owner_id STRING,
+  bucket_name STRING,
+  request_time STRING,
+  request_time_zone STRING,
+  remote_ip STRING,
+  requester STRING,
+  request_id STRING,
+  operation STRING,
+  key STRING,
+  request_uri STRING,
+  http_status STRING,
+  error_code STRING,
+  bytes_sent BIGINT,
+  object_size BIGINT,
+  total_time STRING,
+  turn_around_time STRING,
+  referrer STRING,
+  user_agent STRING,
+  version_id STRING,
+  host_id STRING,
+  signature_version STRING,
+  cipher_suite STRING,
+  authentication_type STRING,
+  host_header STRING,
+  tls_version STRING
+)
+USING csv
+OPTIONS (
+  sep=' '
+)
+LOCATION '{s3_bucket_location}';

server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json

@@ -5,9 +5,23 @@
   "description": "Analyze Amazon S3 server logs.",
   "license": "Apache-2.0",
   "type": "logs_s3",
-  "labels": ["Observability", "Logs", "AWS", "Cloud"],
+  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "S3 Logo",
@@ -39,7 +53,21 @@
       "name": "aws_s3",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
+    },
+    {
+      "name": "create_table",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query"
+    },
+    {
+      "name": "create_mv",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {