feat: schema propagation events in timeline UI (#13892)

teallarson · chandlerprall · teallarson · commit 6c6b65746ac2 · 2024-09-11T14:46:52.000-04:00
Co-authored-by: Chandler Prall &lt;chandler.prall@gmail.com&gt;
diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx
@@ -13,8 +13,8 @@ import { getSortedDiff } from "./utils";
 
 interface CatalogDiffModalProps {
   catalogDiff: CatalogDiff;
-  catalog: AirbyteCatalog;
-  onComplete: () => void;
+  catalog?: AirbyteCatalog;
+  onComplete?: () => void;
 }
 
 export const CatalogDiffModal: React.FC<CatalogDiffModalProps> = ({ catalogDiff, catalog, onComplete }) => {
@@ -32,11 +32,13 @@ export const CatalogDiffModal: React.FC<CatalogDiffModalProps> = ({ catalogDiff,
           {changedItems.length > 0 && <FieldSection streams={changedItems} diffVerb="changed" />}
         </div>
       </ModalBody>
-      <ModalFooter>
-        <Button onClick={onComplete} data-testid="update-schema-confirm-btn">
-          <FormattedMessage id="connection.updateSchema.confirm" />
-        </Button>
-      </ModalFooter>
+      {onComplete && (
+        <ModalFooter>
+          <Button onClick={onComplete} data-testid="update-schema-confirm-btn">
+            <FormattedMessage id="connection.updateSchema.confirm" />
+          </Button>
+        </ModalFooter>
+      )}
     </>
   );
 };
diff --git a/airbyte-webapp/src/components/ui/Modal/Modal.module.scss b/airbyte-webapp/src/components/ui/Modal/Modal.module.scss
@@ -42,6 +42,7 @@
   max-width: calc(100vw - #{variables.$spacing-lg} * 2);
   display: flex;
   flex-direction: column;
+  overflow: hidden;
 
   &.sm {
     width: variables.$width-modal-sm;
diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts
@@ -15,6 +15,7 @@ export interface Experiments {
   "connection.onboarding.sources": string;
   "connection.rateLimitedUI": boolean;
   "connection.timeline": boolean;
+  "connection.timeline.schemaUpdates": boolean;
   "connector.airbyteCloudIpAddresses": string;
   "connector.suggestedSourceConnectors": string;
   "connector.suggestedDestinationConnectors": string;
diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json
@@ -2,6 +2,7 @@
   "general.unicodeBullet": "•",
   "general.dash": "-",
   "general.maskedString": "***********",
+  "general.airbyte": "Airbyte",
   "webapp.cannotReachServer": "Cannot reach server. The server may still be starting up.",
   "notifications.error.health": "Cannot reach server",
   "notifications.error.somethingWentWrong": "Something went wrong",
@@ -758,6 +759,12 @@
   "connection.timeline.connection_settings_update.descriptionWithUserMulti": "{user} changed the {field} from \"{from}\" to \"{to}\" and {otherChanges, plural, one {# other change} other {# other changes}}.",
   "connection.timeline.connection_settings_update.descriptionWithoutUser": "<b>{field}</b> changed from \"{from}\" to \"{to}\"",
   "connection.timeline.connection_settings_update.View details": "View details",
+  "connection.timeline.schema_update": "Schema updated",
+  "connection.timeline.schema_update.description": "{user} applied a change in the schema: {schemaMessage}",
+  "connection.timeline.schema_update.streamsAdded": "{streams, plural, one {# stream added} other {# streams added}}",
+  "connection.timeline.schema_update.streamsRemoved": "{streams, plural, one {# stream removed} other {# streams removed}}",
+  "connection.timeline.schema_update.fieldChanges": "{streams, plural, one {# stream} other {# streams}} changed",
+  "connection.timeline.schema_update.viewDetails": "View details",
 
   "connectionAutoDisabledReason.ONLY_FAILED_JOBS_RECENTLY": "Airbyte disabled the connection due to consecutive failures.",
   "connectionAutoDisabledReason.TOO_MANY_CONSECUTIVE_FAILED_JOBS_IN_A_ROW": "Airbyte disabled the connection due to consecutive failures.",
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/ConnectionTimelineEventIcon.module.scss b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/ConnectionTimelineEventIcon.module.scss
@@ -10,6 +10,11 @@
   height: 30px;
   border-radius: 50%;
   background-color: colors.$grey-100;
+
+  &--lg {
+    width: 40px;
+    height: 40px;
+  }
 }
 
 .connectionTimelineEventIcon__icon {
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/ConnectionTimelineEventIcon.tsx b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/ConnectionTimelineEventIcon.tsx
@@ -8,9 +8,14 @@ export const ConnectionTimelineEventIcon: React.FC<{
   icon: IconProps["type"];
   statusIcon?: IconProps["type"];
   running?: boolean;
-}> = ({ icon, statusIcon, running }) => {
+  size?: "lg" | "sm";
+}> = ({ icon, statusIcon, running, size = "sm" }) => {
   return (
-    <div className={classNames(styles.connectionTimelineEventIcon)}>
+    <div
+      className={classNames(styles.connectionTimelineEventIcon, {
+        [styles["connectionTimelineEventIcon--lg"]]: size === "lg",
+      })}
+    >
       {statusIcon && (
         <div className={styles.connectionTimelineEventIcon__statusIndicator}>
           <Icon
@@ -34,7 +39,7 @@ export const ConnectionTimelineEventIcon: React.FC<{
       {running && !statusIcon && (
         <CircleLoader title="syncing" className={styles.connectionTimelineEventIcon__running} />
       )}
-      <Icon type={icon} size="sm" color="disabled" className={styles.connectionTimelineEventIcon__icon} />
+      <Icon type={icon} size={size} color="disabled" className={styles.connectionTimelineEventIcon__icon} />
     </div>
   );
 };
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/components/EventLineItem.tsx b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/components/EventLineItem.tsx
@@ -3,6 +3,7 @@ import { InferType } from "yup";
 import { Box } from "components/ui/Box";
 
 import { ConnectionEvent } from "core/api/types/AirbyteClient";
+import { useExperiment } from "hooks/services/Experiment";
 
 import { ClearEventItem } from "./ClearEventItem";
 import { ConnectionDisabledEventItem } from "./ConnectionDisabledEventItem";
@@ -11,6 +12,7 @@ import { ConnectionSettingsUpdateEventItem } from "./ConnectionSettingsUpdateEve
 import { JobStartEventItem } from "./JobStartEventItem";
 import { RefreshEventItem } from "./RefreshEventItem";
 import { RunningJobItem } from "./RunningJobItem";
+import { SchemaUpdateEventItem } from "./SchemaUpdateEventItem";
 import { SyncEventItem } from "./SyncEventItem";
 import { SyncFailEventItem } from "./SyncFailEventItem";
 import {
@@ -23,9 +25,12 @@ import {
   connectionEnabledEventSchema,
   connectionDisabledEventSchema,
   connectionSettingsUpdateEventSchema,
+  schemaUpdateEventSchema,
 } from "../types";
 
 export const EventLineItem: React.FC<{ event: ConnectionEvent | InferType<typeof jobRunningSchema> }> = ({ event }) => {
+  const showSchemaUpdates = useExperiment("connection.timeline.schemaUpdates", false);
+
   if (jobRunningSchema.isValidSync(event, { recursive: true, stripUnknown: true })) {
     return (
       <Box py="lg" key={event.id}>
@@ -80,6 +85,13 @@ export const EventLineItem: React.FC<{ event: ConnectionEvent | InferType<typeof
         <ConnectionSettingsUpdateEventItem event={event} />
       </Box>
     );
+  } else if (showSchemaUpdates && schemaUpdateEventSchema.isValidSync(event, { recursive: true, stripUnknown: true })) {
+    return (
+      <Box py="lg" key={event.id}>
+        <SchemaUpdateEventItem event={event} />
+      </Box>
+    );
   }
+
   return null;
 };
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/components/SchemaUpdateEventItem.tsx b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/components/SchemaUpdateEventItem.tsx
@@ -0,0 +1,106 @@
+import { useMemo } from "react";
+import { FormattedDate, FormattedList, FormattedMessage, useIntl } from "react-intl";
+import { InferType } from "yup";
+
+import { CatalogDiffModal } from "components/connection/CatalogDiffModal";
+import { getSortedDiff } from "components/connection/CatalogDiffModal/utils";
+import { Button } from "components/ui/Button";
+import { FlexContainer } from "components/ui/Flex";
+import { Text } from "components/ui/Text";
+
+import { useModalService } from "hooks/services/Modal";
+
+import { TimelineEventUser } from "./TimelineEventUser";
+import { ConnectionTimelineEventActions } from "../ConnectionTimelineEventActions";
+import { ConnectionTimelineEventIcon } from "../ConnectionTimelineEventIcon";
+import { ConnectionTimelineEventItem } from "../ConnectionTimelineEventItem";
+import { ConnectionTimelineEventSummary } from "../ConnectionTimelineEventSummary";
+import { schemaUpdateEventSchema } from "../types";
+import { titleIdMap } from "../utils";
+
+export const SchemaUpdateEventItem: React.FC<{ event: InferType<typeof schemaUpdateEventSchema> }> = ({ event }) => {
+  const titleId = titleIdMap[event.eventType];
+  const { formatMessage } = useIntl();
+  const { openModal } = useModalService();
+
+  const { newItems, removedItems, changedItems } = useMemo(
+    () => getSortedDiff(event.summary.catalogDiff.transforms),
+    [event.summary.catalogDiff.transforms]
+  );
+
+  const schemaMessage = useMemo(() => {
+    const parts = [];
+
+    if (newItems.length > 0) {
+      parts.push(formatMessage({ id: "connection.timeline.schema_update.streamsAdded" }, { streams: newItems.length }));
+    }
+
+    if (removedItems.length > 0) {
+      parts.push(
+        formatMessage({ id: "connection.timeline.schema_update.streamsRemoved" }, { streams: removedItems.length })
+      );
+    }
+    if (changedItems.length > 0) {
+      parts.push(
+        formatMessage({ id: "connection.timeline.schema_update.fieldChanges" }, { streams: changedItems.length })
+      );
+    }
+
+    return (
+      <>
+        <FormattedList value={parts} />.
+      </>
+    );
+  }, [changedItems.length, formatMessage, newItems.length, removedItems.length]);
+
+  const triggerSchemaChangesModal = () =>
+    openModal<void>({
+      title: (
+        <FlexContainer alignItems="center">
+          <ConnectionTimelineEventIcon icon="schema" size="lg" />
+          <FlexContainer gap="xs" direction="column">
+            <Text size="lg">
+              <FormattedMessage id="connection.timeline.schema_update" />
+            </Text>
+            <Text size="lg" color="grey400">
+              <FormattedDate value={event.createdAt * 1000} timeStyle="short" dateStyle="medium" />
+            </Text>
+          </FlexContainer>
+        </FlexContainer>
+      ),
+      size: "md",
+      testId: "catalog-diff-modal",
+      content: () => <CatalogDiffModal catalogDiff={event.summary.catalogDiff} />,
+    });
+
+  return (
+    <ConnectionTimelineEventItem>
+      <ConnectionTimelineEventIcon icon="schema" />
+      <ConnectionTimelineEventSummary>
+        <FlexContainer gap="xs" direction="column">
+          <Text bold>
+            <FormattedMessage id={titleId} />
+          </Text>
+
+          <Text as="span" size="sm" color="grey400">
+            <FormattedMessage
+              id="connection.timeline.schema_update.description"
+              values={{
+                user: !!event.user ? (
+                  <TimelineEventUser user={event.user} />
+                ) : (
+                  <FormattedMessage id="general.airbyte" />
+                ),
+                schemaMessage,
+              }}
+            />{" "}
+            <Button variant="link" onClick={triggerSchemaChangesModal}>
+              <FormattedMessage id="connection.timeline.schema_update.viewDetails" />
+            </Button>
+          </Text>
+        </FlexContainer>
+      </ConnectionTimelineEventSummary>
+      <ConnectionTimelineEventActions createdAt={event.createdAt} />
+    </ConnectionTimelineEventItem>
+  );
+};
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/types.ts b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/types.ts
@@ -6,10 +6,13 @@ import {
   FailureOrigin,
   FailureReason,
   FailureType,
+  FieldTransformTransformType,
   Geography,
   JobConfigType,
   NamespaceDefinitionType,
   NonBreakingChangesPreference,
+  StreamAttributeTransformTransformType,
+  StreamTransformTransformType,
 } from "core/api/types/AirbyteClient";
 
 /**
@@ -38,6 +41,9 @@ const connectionAutoDisabledReasons = [
 ];
 
 // property-specific schemas
+/**
+ * @typedef {import("core/api/types/AirbyteClient").StreamDescriptor}
+ */
 const streamDescriptorSchema = yup.object({
   name: yup.string().required(),
   namespace: yup.string().optional(),
@@ -49,6 +55,80 @@ const jobRunningStreamSchema = yup.object({
   configType: yup.mixed<JobConfigType>().oneOf(["sync", "refresh", "clear", "reset_connection"]).required(),
 });
 
+/**
+ * @typedef {import("core/api/types/AirbyteClient").FieldSchema}
+ */
+const fieldSchema = yup.object({
+  schema: yup.object().optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").FieldSchemaUpdate}
+ */
+const fieldSchemaUpdateSchema = yup.object({
+  newSchema: fieldSchema.required(),
+  oldSchema: fieldSchema.required(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").FieldTransform}
+ */
+const fieldTransformSchema = yup.object({
+  addField: fieldSchema.optional(),
+  breaking: yup.boolean().required(),
+  fieldName: yup.array().of(yup.string()).required(),
+  removeField: fieldSchema.optional(),
+  transformType: yup
+    .mixed<FieldTransformTransformType>()
+    .oneOf(["add_field", "remove_field", "update_field_schema"])
+    .required(),
+  updateFieldSchema: fieldSchemaUpdateSchema.optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").StreamAttributePrimaryKeyUpdate}
+ */
+const streamAttributePrimaryKeyUpdateSchema = yup.object({
+  newPrimaryKey: yup.array().of(yup.array().of(yup.string())).optional(),
+  oldPrimaryKey: yup.array().of(yup.array().of(yup.string())).optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").StreamAttributeTransform}
+ */
+const streamAttributeTransformSchema = yup.object({
+  breaking: yup.boolean().required(),
+  transformType: yup.mixed<StreamAttributeTransformTransformType>().oneOf(["update_primary_key"]).required(),
+  updatePrimaryKey: streamAttributePrimaryKeyUpdateSchema.optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").StreamTransformUpdateStream}
+ */
+const streamTransformUpdateStreamSchema = yup.object({
+  fieldTransforms: yup.array().of(fieldTransformSchema).optional(),
+  streamAttributeTransforms: yup.array().of(streamAttributeTransformSchema).optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").StreamTransform}
+ */
+const streamTransformsSchema = yup.object({
+  streamDescriptor: streamDescriptorSchema.required(),
+  transformType: yup
+    .mixed<StreamTransformTransformType>()
+    .oneOf(["add_stream", "remove_stream", "update_stream"])
+    .required(),
+  updateStream: streamTransformUpdateStreamSchema.optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").CatalogDiff}
+ */
+const catalogDiffSchema = yup.object({
+  transforms: yup.array().of(streamTransformsSchema).required(),
+});
+
 export type TimelineFailureReason = Omit<FailureReason, "timestamp">;
 
 export const jobFailureReasonSchema = yup.object({
@@ -61,6 +141,9 @@ export const jobFailureReasonSchema = yup.object({
   stacktrace: yup.string().optional(),
 });
 
+/**
+ * @typedef {import("core/api/types/AirbyteClient").UserReadInConnectionEvent}
+ */
 export const userInEventSchema = yup.object({
   email: yup.string().optional(),
   id: yup.string().optional(),
@@ -161,6 +244,14 @@ export const connectionSettingsUpdateEventSummarySchema = yup.object({
     .required(),
 });
 
+export const schemaUpdateSummarySchema = yup.object({
+  catalogDiff: catalogDiffSchema.required(),
+  updateReason: yup.mixed().oneOf(["SCHEMA_CHANGE_AUTO_PROPAGATE"]).optional(),
+});
+
+/**
+ * @typedef {import("core/api/types/AirbyteClient").ConnectionEvent}
+ */
 export const generalEventSchema = yup.object({
   id: yup.string().required(),
   connectionId: yup.string().required(),
@@ -241,3 +332,8 @@ export const connectionSettingsUpdateEventSchema = generalEventSchema.shape({
   eventType: yup.mixed<ConnectionEventType>().oneOf([ConnectionEventType.CONNECTION_SETTINGS_UPDATE]).required(),
   summary: connectionSettingsUpdateEventSummarySchema.required(),
 });
+
+export const schemaUpdateEventSchema = generalEventSchema.shape({
+  eventType: yup.mixed<ConnectionEventType>().oneOf([ConnectionEventType.SCHEMA_UPDATE]).required(),
+  summary: schemaUpdateSummarySchema.required(),
+});
diff --git a/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/utils.tsx b/airbyte-webapp/src/pages/connections/ConnectionTimelinePage/utils.tsx
