feat(connector-insights): track manifest usage of $parameters and custom component classes (#43036)

Co-authored-by: Octavia Squidington III <[email protected]>

Author: natikgadzhi

airbyte-ci/connectors/connectors_insights/README.md

@@ -25,7 +25,7 @@ connectors-insights generate --output-directory <path-to-local-output-dir> --gcs
 ### CLI Options
 
 - `generate`: The command to generate the artifacts.
-  
+
 - `-o, --output-dir`: Specifies the local directory where the generated artifacts will be saved. In this example, artifacts are saved to `/Users/augustin/Desktop/insights`.
 
 - `-g, --gcs-uri`: The Google Cloud Storage (GCS) URI prefix where the artifacts will be uploaded. In the form: `gs://<bucket>/<key-prefix>`.
@@ -56,6 +56,9 @@ This CLI is currently running nightly in GitHub Actions. The workflow can be fou
 
 ## Changelog
 
+### 0.3.0
+Adding `manifest_uses_parameters`, `manifest_uses_custom_components`, and `manifest_custom_components_classes` insights.
+
 ### 0.2.4
 Do not generate insights for `*-scaffold-*` and `*-strict-encrypt` connectors.
 

airbyte-ci/connectors/connectors_insights/poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "connectors-insights"
-version = "0.2.4"
+version = "0.3.0"
 description = ""
 authors = ["Airbyte <[email protected]>"]
 readme = "README.md"
@@ -31,6 +31,9 @@ mypy = "^1.8.0"
 types-requests = "^2.32.0.20240602"
 types-beautifulsoup4 = "^4.12.0.20240511"
 
+[tool.ruff]
+line-length = 140
+
 [tool.ruff.lint]
 select = ["F"]
 

airbyte-ci/connectors/connectors_insights/pyproject.toml

@@ -83,7 +83,6 @@ async def generate(
     concurrency: int,
     rewrite: bool,
 ) -> None:
-
     logger = logging.getLogger(__name__)
     result_backends: List[ResultBackend] = []
     if output_directory:

airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py

@@ -1,5 +1,4 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
 from __future__ import annotations
 
 import datetime
@@ -14,6 +13,7 @@
 from connectors_insights.pylint import get_pylint_output
 from connectors_insights.result_backends import FileToPersist, ResultBackend
 from connectors_insights.sbom import get_json_sbom
+from typing_extensions import Mapping
 
 if TYPE_CHECKING:
     from typing import Dict, List, Tuple
@@ -23,6 +23,19 @@
     from connector_ops.utils import Connector  # type: ignore
 
 
+def get_manifest_inferred_insights(connector: Connector) -> dict:
+    manifest = connector.manifest_path.read_text()
+
+    schemas_directory = connector.code_path / connector.technical_name.replace("-", "_") / "schemas"
+
+    return {
+        "manifest_uses_parameters": manifest.find("$parameters") != -1,
+        "manifest_uses_custom_components": manifest.find("class_name:") != -1,
+        "manifest_custom_component_classes": re.findall(r"class_name: (.+)", manifest),
+        "has_json_schemas": schemas_directory.is_dir() and any(schemas_directory.iterdir()),
+    }
+
+
 def get_metadata_inferred_insights(connector: Connector) -> Dict:
     return {
         "connector_technical_name": connector.technical_name,
@@ -155,6 +168,7 @@ def generate_insights(connector: Connector, sbom: str | None, pylint_output: str
     return ConnectorInsights(
         **{
             **get_metadata_inferred_insights(connector),
+            **get_manifest_inferred_insights(connector),
             **get_pylint_inferred_insights(pylint_output),
             **get_sbom_inferred_insights(sbom, connector),
             "ci_on_master_report": ci_on_master_report,

airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py

@@ -34,6 +34,10 @@ class ConnectorInsights(BaseModel):
     deprecated_classes_in_use: list[str] | None
     deprecated_modules_in_use: list[str] | None
     forbidden_method_names_in_use: list[str] | None
+    manifest_uses_parameters: StrictBool
+    manifest_uses_custom_components: StrictBool
+    manifest_custom_component_classes: list[str] | None
+    has_json_schemas: StrictBool
 
     class Config:
         json_encoders = {dict: lambda v: json.dumps(v, default=pydantic_encoder)}

airbyte-ci/connectors/connectors_insights/src/connectors_insights/models.py

@@ -10,7 +10,6 @@
 from connectors_insights.utils import never_fail_exec
 
 if TYPE_CHECKING:
-
     import dagger
     from connector_ops.utils import Connector  # type: ignore
 

airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py

@@ -1,5 +1,4 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
 from __future__ import annotations
 
 import logging
@@ -65,7 +64,6 @@ def artifact_already_exists(self, connector: Connector, file_to_persist: FileToP
 
 
 class GCSBucket(ResultBackend):
-
     DEFAULT_GCP_PROJECT = "prod-ab-cloud-proj"
 
     def __init__(self, bucket_name: str, key_prefix: str, gcp_project: str = DEFAULT_GCP_PROJECT):

airbyte-ci/connectors/connectors_insights/src/connectors_insights/result_backends.py

@@ -6,7 +6,6 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-
     import dagger
     from connector_ops.utils import Connector  # type: ignore
 

airbyte-ci/connectors/connectors_insights/src/connectors_insights/sbom.py

@@ -7,7 +7,8 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Set, Tuple, List, Callable
+    from typing import Callable, List, Set, Tuple
+
     from dagger import Container
 
 from connector_ops.utils import METADATA_FILE_NAME, Connector  # type: ignore