Add CloudFront queries for integrations and integration table bug fix (#1687)

* initial commit for adding cloudfront integrations

Signed-off-by: Sean Li <[email protected]>

* adding queries and fixing bug in integrations table

Signed-off-by: Sean Li <[email protected]>

* updating ndjson, changing options

Signed-off-by: Sean Li <[email protected]>

* removing OSD bump

Signed-off-by: Sean Li <[email protected]>

* updating tests

Signed-off-by: Sean Li <[email protected]>

* updating MV name

Signed-off-by: Sean Li <[email protected]>

---------

Signed-off-by: Sean Li <[email protected]>

Author: sejli

server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/aws_cloudfront-1.0.0.ndjson

@@ -0,0 +1,42 @@
+CREATE MATERIALIZED VIEW {table_name}__mview AS
+SELECT
+  to_timestamp(trim(BOTH '[]' FROM concat(date, ' ', time)), 'yyyy-MM-dd HH:mm:ss') AS `@timestamp`,
+  c_ip AS `aws.cloudfront.c-ip`,
+  c_port as `aws.cloudfront.c-port`,
+  cs_cookie as `aws.cloudfront.cs-cookie`,
+  cs_host as `aws.cloudfront.cs-host`,
+  cs_referrer as `aws.cloudfront.cs-referer`,
+  cs_user_agent as `aws.cloudfront.cs-user-agent`,
+  cs_bytes as `aws.cloudfront.cs-bytes`,
+  cs_method as `aws.cloudfront.cs-method`,
+  cs_protocol as `aws.cloudfront.cs-protocol`,
+  cs_protocol_version as `aws.cloudfront.cs-protocol-version`,
+  cs_uri_query as `aws.cloudfront.cs-uri-query`,
+  cs_uri_stem as `aws.cloudfront.cs-uri-stem`,
+  fle_encrypted_fields as `aws.cloudfront.fle-encrypted-fields`,
+  fle_status as `aws.cloudfront.fle-status`,
+  sc_bytes as `aws.cloudfront.sc-bytes`,
+  sc_content_len as `aws.cloudfront.sc-content-len`,
+  sc_content_type as `aws.cloudfront.sc-content-type`,
+  sc_range_end as `aws.cloudfront.sc-range-end`,
+  sc_range_start as `aws.cloudfront.sc-range-start`,
+  sc_status as `aws.cloudfront.sc-status`,
+  ssl_cipher as `aws.cloudfront.ssl-cipher`,
+  ssl_protocol as `aws.cloudfront.ssl-protocol`,
+  time_taken as `aws.cloudfront.time-taken`,
+  time_to_first_byte as `aws.cloudfront.time-to-first-byte`,
+  x_edge_detailed_result_type as `aws.cloudfront.x-edge_detailed-result-type`,
+  x_edge_location as `aws.cloudfront.x-edge-location`,
+  x_edge_request_id as `aws.cloudfront.x-edge-request-id`,
+  x_edge_result_type as `aws.cloudfront.x-edge-result-type`,
+  x_edge_response_result_type as `aws.cloudfront.x-edge-response-result-type`,
+  x_forwarded_for as `aws.cloudfront.x-forwarded-for`,
+  x_host_header as `aws.cloudfront.x-host-header`
+FROM 
+  {table_name}
+WITH (
+  auto_refresh = true,
+  checkpoint_location = '{s3_checkpoint_location}',
+  watermark_delay = '1 Minute',
+  extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);

server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql

@@ -0,0 +1,39 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+  `date` STRING,
+  time STRING,
+  x_edge_location STRING,
+  sc_bytes BIGINT,
+  c_ip STRING,
+  cs_method STRING,
+  cs_host STRING,
+  cs_uri_stem STRING,
+  sc_status INT,
+  cs_referrer STRING,
+  cs_user_agent STRING,
+  cs_uri_query STRING,
+  cs_cookie STRING,
+  x_edge_result_type STRING,
+  x_edge_request_id STRING,
+  x_host_header STRING,
+  cs_protocol STRING,
+  cs_bytes BIGINT,
+  time_taken FLOAT,
+  x_forwarded_for STRING,
+  ssl_protocol STRING,
+  ssl_cipher STRING,
+  x_edge_response_result_type STRING,
+  cs_protocol_version STRING,
+  fle_status STRING,
+  fle_encrypted_fields INT,
+  c_port INT,
+  time_to_first_byte FLOAT,
+  x_edge_detailed_result_type STRING,
+  sc_content_type STRING,
+  sc_content_len BIGINT,
+  sc_range_start BIGINT,
+  sc_range_end BIGINT
+)
+USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS ( sep='\t' )
+TBLPROPERTIES ( 'skip.header.line.count'='2' );

server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql

@@ -5,7 +5,7 @@
   "description": "Analyze access logs for Amazon CloudFront.",
   "license": "Apache-2.0",
   "type": "logs-aws_cloudfront",
-  "labels": ["Observability", "Logs", "AWS", "Cloud"],
+  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info",
   "statics": {
@@ -20,6 +20,20 @@
       }
     ]
   },
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "components": [
     {
       "name": "aws_cloudfront",
@@ -43,7 +57,21 @@
       "name": "aws_cloudfront",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
+    },
+    {
+      "name": "create_table",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query"
+    },
+    {
+      "name": "create_mv",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {

server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json

@@ -196,7 +196,10 @@ describe('IntegrationsKibanaBackend', () => {
 
       const result = await backend.getIntegrationInstances();
 
-      expect(mockSavedObjectsClient.find).toHaveBeenCalledWith({ type: 'integration-instance' });
+      expect(mockSavedObjectsClient.find).toHaveBeenCalledWith({
+        type: 'integration-instance',
+        perPage: 1000,
+      });
       expect(result).toEqual({
         total: findResult.total,
         hits: savedObjects.map((obj) => ({ id: obj.id, ...obj.attributes })),

server/adaptors/integrations/__test__/manager.test.ts

@@ -93,7 +93,7 @@ export class IntegrationsManager implements IntegrationsAdaptor {
     _query?: IntegrationInstanceQuery
   ): Promise<IntegrationInstancesSearchResult> => {
     addRequestToMetric('integrations', 'get', 'count');
-    const result = await this.client.find({ type: 'integration-instance' });
+    const result = await this.client.find({ type: 'integration-instance', perPage: 1000 });
     return Promise.resolve({
       total: result.total,
       hits: result.saved_objects?.map((x) => ({