Show version and changelog status for affected connectors

.github/comment_templates/connector_dependency_template.md

@@ -1,25 +1,54 @@
 <!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
 
-NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to run corresponding integration tests:
+NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do the following as needed:
+- Run integration tests
+- Bump connector version
+- Add changelog
+- Publish the new version
 
-<details>
+<details {source_open}>
 <summary>
-Sources ({num_sources})
+
+### {source_status_summary} Sources ({num_sources})
 
 </summary>
 
-{sources}
+| Connector | Version | Changelog | Publish |
+| --- | :---: | :---: | :---: |
+{source_rows}
 
 </details>
 
-<details>
+<details {destination_open}>
 <summary>
-Destinations ({num_destinations})
+
+### {destination_status_summary} Destinations ({num_destinations})
 
 </summary>
 
-{destinations}
+| Connector | Version | Changelog | Publish |
+| --- | :---: | :---: | :---: |
+{destination_rows}
 
 </details>
 
 {others}
+
+<details>
+
+<summary>
+
+### Notes
+
+</summary>
+
+| Category | Status | Actionable Item |
+| --- | :---: | --- |
+| Version | ❌<br/>mismatch | The version of the connector is different from its normal variant. Please bump the version of the connector. |
+| | ⚠<br/>doc not found | The connector does not seem to have a documentation file. This can be normal (e.g. basic connector like `source-jdbc` is not published or documented). Please double-check to make sure that it is not a bug. |
+| Changelog | ⚠<br/>doc not found | The connector does not seem to have a documentation file. This can be normal (e.g. basic connector like `source-jdbc` is not published or documented). Please double-check to make sure that it is not a bug. |
+| | ❌<br/>changelog missing | There is no chnagelog for the current version of the connector. If you are the author of the current version, please add a changelog. |
+| Publish | ⚠<br/>not in seed | The connector is not in the seed file (e.g. `source_definitions.yaml`), so its publication status cannot be checked. This can be normal (e.g. some connectors are cloud-specific, and only listed in the cloud seed file). Please double-check to make sure that it is not a bug. |
+| | ❌<br/>diff seed version | The connector exists in the seed file, but the latest version is not listed there. This usually means that the latest version is not published. Please use the `/publish` command to publish the latest version. |
+
+</details>

tools/bin/ci_check_dependency.py

@@ -1,9 +1,12 @@
 import sys
 import os
 import os.path
+import yaml
 
-CONNECTOR_FOLDER = "airbyte-integrations/connectors/"
 CONNECTOR_PATH = "./airbyte-integrations/connectors/"
+DOC_PATH = "docs/integrations/"
+SOURCE_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/source_definitions.yaml"
+DESTINATION_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/destination_definitions.yaml"
 IGNORE_LIST = [
     # Java
     "/src/test/","/src/test-integration/", "/src/testFixtures/",
@@ -81,19 +84,81 @@ def get_depended_connectors(changed_modules, all_build_gradle_files):
     depended_connectors = []
     for changed_module in changed_modules:
         for connector, gradle_file in all_build_gradle_files.items():
+            if gradle_file is None:
+                continue
             with open(gradle_file) as file:
                 if changed_module in file.read():
                     depended_connectors.append(connector)
     return depended_connectors
 
 
+def get_connector_version(connector):
+    with open(f"{CONNECTOR_PATH}/{connector}/Dockerfile") as f:
+        for line in f:
+            if "io.airbyte.version" in line:
+                return line.split("=")[1].strip()
+
+
+def get_connector_version_status(connector, version):
+    if "strict-encrypt" not in connector:
+        return f"`{version}`"
+    if connector == "source-mongodb-strict-encrypt":
+        base_variant_version = get_connector_version("source-mongodb-v2")
+    else:
+        base_variant_version = get_connector_version(connector.replace("-strict-encrypt", ""))
+    if base_variant_version == version:
+        return f"`{version}`"
+    else:
+        return f"❌ `{version}`<br/>(mismatch: `{base_variant_version}`)"
+
+
+def get_connector_changelog_status(connector, version):
+    type, name = connector.replace("-strict-encrypt", "").split("-", 1)
+    doc_path = f"{DOC_PATH}{type}s/{name}.md"
+    if not os.path.exists(doc_path):
+        return "⚠<br/>(doc not found)"
+    with open(doc_path) as f:
+        after_changelog = False
+        for line in f:
+            if "# changelog" in line.lower():
+                after_changelog = True
+            if after_changelog and version in line:
+                return "✅"
+    return "❌<br/>(changelog missing)"
+
 def as_bulleted_markdown_list(items):
     text = ""
     for item in items:
         text += f"- {item}\n"
     return text
 
 
+def as_markdown_table_rows(connectors, definitions):
+    text = ""
+    for connector in connectors:
+        version = get_connector_version(connector)
+        version_status = get_connector_version_status(connector, version)
+        changelog_status = get_connector_changelog_status(connector, version)
+        definition = next((x for x in definitions if x["dockerRepository"].endswith(connector)), None)
+        if definition is None:
+            publish_status = "⚠<br/>(not in seed)"
+        elif definition["dockerImageTag"] == version:
+            publish_status = "✅"
+        else:
+            publish_status = "❌<br/>(diff seed version)"
+        text += f"| `{connector}` | {version_status} | {changelog_status} | {publish_status} |\n"
+    return text
+
+
+def get_status_summary(rows):
+    if "❌" in rows:
+        return "❌"
+    elif "⚠" in rows:
+        return "⚠"
+    else:
+        return "✅"
+
+
 def write_report(depended_connectors):
     affected_sources = []
     affected_destinations = []
@@ -109,14 +174,35 @@ def write_report(depended_connectors):
     with open(COMMENT_TEMPLATE_PATH, "r") as f:
         template = f.read()
 
+    source_definitions = []
+    destination_definitions = []
+    with open(SOURCE_DEFINITIONS_PATH, 'r') as stream:
+        source_definitions = yaml.safe_load(stream)
+    with open(DESTINATION_DEFINITIONS_PATH, 'r') as stream:
+        destination_definitions = yaml.safe_load(stream)
+
     others_md = ""
     if affected_others:
         others_md += "The following were also affected:\n"
         others_md += as_bulleted_markdown_list(affected_others)
 
+    affected_sources.sort()
+    affected_destinations.sort()
+    affected_others.sort()
+
+    source_rows = as_markdown_table_rows(affected_sources, source_definitions)
+    destination_rows = as_markdown_table_rows(affected_destinations, destination_definitions)
+
+    source_status_summary = get_status_summary(source_rows)
+    destination_status_summary = get_status_summary(destination_rows)
+
     comment = template.format(
-        sources=as_bulleted_markdown_list(affected_sources),
-        destinations=as_bulleted_markdown_list(affected_destinations),
+        source_open="open" if source_status_summary == "❌" else "closed",
+        destination_open="open" if destination_status_summary == "❌" else "closed",
+        source_status_summary=source_status_summary,
+        destination_status_summary=destination_status_summary,
+        source_rows=source_rows,
+        destination_rows=destination_rows,
         others=others_md,
         num_sources=len(affected_sources),
         num_destinations=len(affected_destinations)