feat: connector resource configuration UI (#15021)

Co-authored-by: Lake Mossman <[email protected]>

Author: teallarson

airbyte-webapp/src/area/connector/components/ResourceAllocationMenu.tsx

@@ -0,0 +1,174 @@
+import uniq from "lodash/uniq";
+import uniqueId from "lodash/uniqueId";
+import { useCallback, useState } from "react";
+import { useFormContext } from "react-hook-form";
+import { FormattedMessage } from "react-intl";
+
+import { LabelInfo } from "components";
+import { ControlLabels } from "components/LabeledControl";
+import { Box } from "components/ui/Box";
+import { FlexContainer } from "components/ui/Flex";
+import { ListBox } from "components/ui/ListBox";
+import { Text } from "components/ui/Text";
+
+import { ConnectorDefinition } from "core/domain/connector";
+import { isSourceDefinition } from "core/domain/connector/source";
+import { FeatureItem, useFeature } from "core/services/features";
+import { ConnectorFormValues } from "views/Connector/ConnectorForm";
+import { PropertyError } from "views/Connector/ConnectorForm/components/Property/PropertyError";
+import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormContext";
+
+export const API_RESOURCES = {
+  default: {
+    memory: 2,
+    cpu: 1,
+  },
+  large: {
+    memory: 3,
+    cpu: 2,
+  },
+  memoryIntensive: {
+    memory: 6,
+    cpu: 2,
+  },
+  maximum: {
+    memory: 8,
+    cpu: 4,
+  },
+};
+
+export const DB_RESOURCES = {
+  default: {
+    memory: 2,
+    cpu: 2,
+  },
+  large: {
+    memory: 3,
+    cpu: 3,
+  },
+  memoryIntensive: {
+    memory: 6,
+    cpu: 3,
+  },
+  maximum: {
+    memory: 8,
+    cpu: 4,
+  },
+};
+
+const getOptions = (connectorType: "api" | "database") => {
+  const values = connectorType === "api" ? API_RESOURCES : DB_RESOURCES;
+
+  return Object.entries(values).map(([size, value]) => {
+    return {
+      value,
+      label: (
+        <Text>
+          <FormattedMessage
+            id="form.resourceAllocation.label"
+            values={{
+              size,
+              cpu: value.cpu,
+              memory: value.memory,
+              details: (...details: React.ReactNode[]) => (
+                <Text color="grey" as="span">
+                  {details}
+                </Text>
+              ),
+            }}
+          />
+        </Text>
+      ),
+    };
+  });
+};
+
+export const getConnectorType = (selectedConnectorDefinition?: ConnectorDefinition) => {
+  if (!selectedConnectorDefinition) {
+    return undefined;
+  }
+
+  return isSourceDefinition(selectedConnectorDefinition)
+    ? selectedConnectorDefinition.sourceType === "file"
+      ? "database"
+      : selectedConnectorDefinition.sourceType === "custom"
+      ? "api"
+      : selectedConnectorDefinition.sourceType
+    : "database";
+};
+
+export const useConnectorResourceAllocation = () => {
+  const supportsResourceAllocation = useFeature(FeatureItem.ConnectorResourceAllocation);
+
+  const isHiddenResourceAllocationField = useCallback(
+    (fieldPath: string) => {
+      // we want to hide the resourceAllocation sub-fields
+      return supportsResourceAllocation && fieldPath.startsWith("resourceAllocation.");
+    },
+    [supportsResourceAllocation]
+  );
+  return { isHiddenResourceAllocationField };
+};
+
+/**
+ * TODO: when we add support for reading the current value from the SourceRead/DestinationRead
+ * we will want to (a) map from the current value to one of the selectable options AND ALSO
+ * (b) support values that do not fit the preconfigured options (and just show them in the UI).
+ * We do not need to support a custom input yet, though.
+ */
+export const ResourceAllocationMenu: React.FC = () => {
+  const { selectedConnectorDefinition } = useConnectorForm();
+  const [controlId] = useState(`resource-listbox-control-${uniqueId()}`);
+
+  const { getValues, setValue, formState, getFieldState } = useFormContext<ConnectorFormValues>();
+  const meta = getFieldState("resourceAllocation", formState);
+
+  const connectorType = getConnectorType(selectedConnectorDefinition);
+  if (!connectorType) {
+    return null;
+  }
+
+  const options = getOptions(connectorType);
+
+  const errorMessage = Array.isArray(meta.error) ? (
+    <FlexContainer direction="column" gap="none">
+      {uniq(meta.error.map((error) => error?.message).filter(Boolean)).map((errorMessage, index) => {
+        return <PropertyError key={index}>{errorMessage}</PropertyError>;
+      })}
+    </FlexContainer>
+  ) : !!meta.error?.message ? (
+    <PropertyError>{meta.error.message}</PropertyError>
+  ) : null;
+
+  return (
+    <Box pb="xl" data-testid="resourceAllocationMenu">
+      <ControlLabels
+        htmlFor={controlId}
+        label={<FormattedMessage id="form.resourceAllocation" />}
+        optional
+        infoTooltipContent={
+          <LabelInfo
+            label={<FormattedMessage id="form.resourceAllocation" />}
+            description={
+              <FlexContainer direction="column" gap="sm">
+                <Text inverseColor>
+                  <FormattedMessage id="form.resourceAllocation.tooltip" />
+                </Text>
+                <Text inverseColor>
+                  <FormattedMessage id="form.resourceAllocation.tooltip.note" />
+                </Text>
+              </FlexContainer>
+            }
+          />
+        }
+      />
+      <ListBox
+        id={controlId}
+        options={options}
+        selectedValue={getValues("resourceAllocation")}
+        onSelect={(selectedValue) => setValue("resourceAllocation", selectedValue, { shouldDirty: true })}
+      />
+      {errorMessage}
+    </Box>
+  );
+};

airbyte-webapp/src/core/services/features/types.tsx

@@ -14,6 +14,7 @@ export enum FeatureItem {
   CloudForTeamsUpsell = "CLOUD_FOR_TEAMS_UPSELLING",
   ConnectionHistoryGraphs = "CONNECTION_HISTORY_GRAPHS",
   ConnectorBreakingChangeDeadlines = "CONNECTOR_BREAKING_CHANGE_DEADLINES",
+  ConnectorResourceAllocation = "CONNECTOR_RESOURCE_ALLOCATION",
   DiagnosticsExport = "DIAGNOSTICS_EXPORT",
   DisplayOrganizationUsers = "DISPLAY_ORGANIZATION_USERS",
   EmailNotifications = "EMAIL_NOTIFICATIONS",

airbyte-webapp/src/locales/en.json

@@ -101,6 +101,10 @@
   "form.connectionName.subtitle": "Name for your connection",
   "form.sourceName": "Source name",
   "form.destinationName": "Destination name",
+  "form.resourceAllocation": "Connector Resource Allocation",
+  "form.resourceAllocation.label": "{size, select, default {Default} large {Large} memoryIntensive {Memory intensive} maximum {Maximum} other {Default}} <details>({cpu}CPU, {memory}GB memory)</details>",
+  "form.resourceAllocation.tooltip": "Resources allocated to this connector for a sync. ",
+  "form.resourceAllocation.tooltip.note": "If no nodes with these resources are available during a sync, the sync will enter a pending state and resume when one is free.",
   "form.sourceName.message": "Pick a name to help you identify this source in Airbyte",
   "form.destinationName.message": "Pick a name to help you identify this destination in Airbyte",
   "form.connectionName.message": "Pick a name to help you identify this connection",
@@ -198,6 +202,7 @@
   "form.wait": "Please wait a little bit more…",
   "form.optional": "Optional",
   "form.optionalFields": "Optional fields",
+  "form.advanced": "Advanced",
   "form.min.error": "Must be greater than or equal to {min}",
   "form.max.error": "Must be less than or equal to {max}",
 

airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx

@@ -11,6 +11,7 @@ import { useCompleteOAuth } from "core/api";
 import { DestinationDefinitionSpecificationRead, OAuthConsentRead } from "core/api/types/AirbyteClient";
 import { ConnectorDefinition, ConnectorDefinitionSpecificationRead } from "core/domain/connector";
 import { AirbyteJSONSchema } from "core/jsonSchema/types";
+import { FeatureItem } from "core/services/features";
 import { ConnectorForm } from "views/Connector/ConnectorForm";
 
 import { ConnectorFormValues } from "./types";
@@ -280,10 +281,12 @@ describe("Connector form", () => {
     formValuesOverride,
     propertiesOverride,
     specificationOverride,
+    features,
   }: {
     formValuesOverride?: Record<string, unknown>;
     specificationOverride?: Partial<ConnectorDefinitionSpecificationRead>;
     propertiesOverride?: Record<string, AirbyteJSONSchema>;
+    features?: FeatureItem[];
   } = {}) {
     const renderResult = await render(
       <ConnectorForm
@@ -311,7 +314,8 @@ describe("Connector form", () => {
           } as unknown as DestinationDefinitionSpecificationRead
         }
       />,
-      undefined
+      undefined,
+      features ? [...features] : undefined
     );
     return renderResult.container;
   }
@@ -424,19 +428,25 @@ describe("Connector form", () => {
           additional_same_group: { type: "string" },
         },
       });
-      expect(getInputByName(container, "connectionConfiguration.additional_separate_group")).not.toBeVisible();
-      expect(getInputByName(container, "connectionConfiguration.additional_same_group")).not.toBeVisible();
 
-      await userEvent.click(screen.getAllByTestId("optional-fields").at(0)!);
+      const additionalSeparateGroupInput = getInputByName(
+        container,
+        "connectionConfiguration.additional_separate_group"
+      );
+      const additionalSameGroupInput = getInputByName(container, "connectionConfiguration.additional_same_group");
 
-      expect(getInputByName(container, "connectionConfiguration.additional_separate_group")).toBeVisible();
+      const optionalFields = screen.getAllByTestId("optional-fields");
+      expect(additionalSameGroupInput).not.toBeVisible();
 
-      await userEvent.click(screen.getAllByTestId("optional-fields").at(1)!);
+      await userEvent.click(optionalFields.at(0)!);
+      expect(additionalSameGroupInput).toBeVisible();
 
-      expect(getInputByName(container, "connectionConfiguration.additional_same_group")).toBeVisible();
+      expect(additionalSeparateGroupInput).not.toBeVisible();
+      await userEvent.click(optionalFields.at(1)!);
+      expect(additionalSeparateGroupInput).toBeVisible();
 
-      const input1 = getInputByName(container, "connectionConfiguration.additional_same_group");
-      const input2 = getInputByName(container, "connectionConfiguration.additional_separate_group");
+      const input1 = additionalSameGroupInput;
+      const input2 = additionalSeparateGroupInput;
       await userEvent.type(input1!, "input1");
       await userEvent.type(input2!, "input2");
 
@@ -1110,4 +1120,47 @@ describe("Connector form", () => {
       expect(getOAuthButton(container)).toBeInTheDocument();
     });
   });
+
+  describe("Advanced section", () => {
+    it("does not render the advanced section if Resource Allocation is off + spec does not define it", async () => {
+      await renderForm({});
+
+      expect(screen.queryAllByTestId("optional-fields")).toHaveLength(0);
+    });
+
+    it("renders the Advanced section if the flag is on and value is present in spec without groups", async () => {
+      await renderForm({ features: [FeatureItem.ConnectorResourceAllocation] });
+
+      const optionalFields = screen.getAllByTestId("optional-fields");
+      expect(optionalFields[0]).toHaveTextContent("Advanced");
+      expect(optionalFields).toHaveLength(1);
+    });
+
+    it("renders the Advanced section  if the flag is on and value is present in spec with groups", async () => {
+      await renderForm({
+        propertiesOverride: {
+          additional_separate_group: { type: "string", group: "abc" },
+        },
+        features: [FeatureItem.ConnectorResourceAllocation],
+      });
+
+      const optionalFields = screen.getAllByTestId("optional-fields");
+      expect(optionalFields[0]).toHaveTextContent("Optional fields");
+      expect(optionalFields[1]).toHaveTextContent("Advanced");
+      expect(optionalFields).toHaveLength(2);
+    });
+
+    it("combines resource allocation Advanced section with one described in the spec", async () => {
+      await renderForm({
+        propertiesOverride: {
+          additional_separate_group: { type: "string", group: "advanced" },
+        },
+        features: [FeatureItem.ConnectorResourceAllocation],
+      });
+
+      const optionalFields = screen.getAllByTestId("optional-fields");
+      expect(optionalFields[0]).toHaveTextContent("Advanced");
+      expect(optionalFields).toHaveLength(1);
+    });
+  });
 });

airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx

@@ -8,6 +8,7 @@ import {
   ConnectorDefinitionSpecificationRead,
   SourceDefinitionSpecificationDraft,
 } from "core/domain/connector";
+import { FeatureItem, useFeature } from "core/services/features";
 import { removeEmptyProperties } from "core/utils/form";
 import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker";
 
@@ -35,6 +36,7 @@ export interface ConnectorFormProps extends Omit<FormRootProps, "formFields" | "
 export const ConnectorForm: React.FC<ConnectorFormProps> = (props) => {
   const formId = useUniqueFormId(props.formId);
   const { clearFormChange } = useFormChangeTrackerService();
+  const isResourceAllocationEnabled = useFeature(FeatureItem.ConnectorResourceAllocation);
 
   const {
     formType,
@@ -51,6 +53,7 @@ export const ConnectorForm: React.FC<ConnectorFormProps> = (props) => {
     Boolean(isEditMode),
     formType,
     selectedConnectorDefinitionSpecification,
+    isResourceAllocationEnabled,
     formValues
   );
 

airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/FormSection.tsx

@@ -7,6 +7,7 @@ import { FlexContainer, FlexItem } from "components/ui/Flex";
 import { Heading } from "components/ui/Heading";
 
 import { ArrayOfObjectsSection } from "area/connector/components/ArrayOfObjectsSection";
+import { ResourceAllocationMenu } from "area/connector/components/ResourceAllocationMenu";
 import { FormBlock, GroupDetails } from "core/form/types";
 
 import { AuthSection } from "./auth/AuthSection";
@@ -94,6 +95,7 @@ export const FormSection: React.FC<FormSectionProps> = ({
                 where the formField._type is formCondition, which will be the "outside rendering".
                */}
               {shouldShowAuthButton(sectionPath) && formField._type !== "formCondition" && <AuthSection />}
+              {sectionPath === "resourceAllocation" && <ResourceAllocationMenu />}
               <FormNode sectionPath={sectionPath} formField={formField} disabled={disabled} />
             </React.Fragment>
           );

airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/useGroupsAndSections.test.ts

@@ -63,10 +63,10 @@ function section(blocks: FormBlock[], displayType: DisplayType = "expanded"): Se
   };
 }
 
-const isHiddenAuthField = jest.fn(() => false);
+const isHiddenField = jest.fn(() => false);
 
 function generate(blocks: FormBlock | FormBlock[], groups: GroupDetails[] = []) {
-  return generateGroupsAndSections(blocks, groups, true, isHiddenAuthField);
+  return generateGroupsAndSections(blocks, groups, true, isHiddenField);
 }
 
 describe("useGroupsAndSections", () => {