diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts new file mode 100644 index 0000000000000..6c9a37a856794 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/api.ts @@ -0,0 +1,55 @@ +import { apiOverride } from "core/request/apiOverride"; + +/** + * Get the available dbt Cloud jobs associated with the given workspace config. + */ +export interface WorkspaceGetDbtJobsRequest { + workspaceId: WorkspaceId; + /** The config id associated with the dbt Cloud config, references the webhookConfigId in the core API. */ + dbtConfigId: string; +} + +/** + * The available dbt Cloud jobs for the requested workspace config + */ +export interface WorkspaceGetDbtJobsResponse { + availableDbtJobs: DbtCloudJobInfo[]; +} + +/** + * A dbt Cloud job + */ +export interface DbtCloudJobInfo { + /** The account id associated with the job */ + accountId: number; + /** The the specific job id returned by the dbt Cloud API */ + jobId: number; + /** The human-readable name of the job returned by the dbt Cloud API */ + jobName: string; +} + +/** + * @summary Calls the dbt Cloud `List Accounts` and `List jobs` APIs to get the list of available jobs for the dbt auth token associated with the requested workspace config. + */ +export const webBackendGetAvailableDbtJobsForWorkspace = ( + workspaceGetDbtJobsRequest: WorkspaceGetDbtJobsRequest, + options?: SecondParameter +) => { + return apiOverride( + { + url: `/v1/web_backend/cloud_workspaces/get_available_dbt_jobs`, + method: "post", + headers: { "Content-Type": "application/json" }, + data: workspaceGetDbtJobsRequest, + }, + options + ); +}; + +/** + * Workspace Id from OSS Airbyte instance + */ +export type WorkspaceId = string; + +// eslint-disable-next-line +type SecondParameter any> = T extends (config: any, args: infer P) => any ? P : never; diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts new file mode 100644 index 0000000000000..d158c57640119 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/lib/domain/dbtCloud/index.ts @@ -0,0 +1 @@ +export * from "./api"; diff --git a/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts b/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts index 4504a91220a7f..7ddfe49e124c3 100644 --- a/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts +++ b/airbyte-webapp/src/packages/cloud/services/dbtCloud.ts @@ -9,18 +9,28 @@ // - custom domains aren't yet supported import isEmpty from "lodash/isEmpty"; -import { useMutation } from "react-query"; +import { useMutation, useQuery } from "react-query"; import { OperatorType, WebBackendConnectionRead, OperationRead, WebhookConfigRead } from "core/request/AirbyteClient"; import { useWebConnectionService } from "hooks/services/useConnectionHook"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; +import { + DbtCloudJobInfo, + webBackendGetAvailableDbtJobsForWorkspace, + WorkspaceGetDbtJobsResponse, +} from "packages/cloud/lib/domain/dbtCloud/api"; +import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { useUpdateWorkspace } from "services/workspaces/WorkspacesService"; +import { useConfig } from "./config"; + export interface DbtCloudJob { account: string; job: string; operationId?: string; + jobName?: string; } +export type { DbtCloudJobInfo } from "packages/cloud/lib/domain/dbtCloud/api"; const dbtCloudDomain = "https://cloud.getdbt.com"; const webhookConfigName = "dbt cloud"; const executionBody = `{"cause": "airbyte"}`; @@ -28,26 +38,35 @@ const jobName = (t: DbtCloudJob) => `${t.account}/${t.job}`; const isDbtWebhookConfig = (webhookConfig: WebhookConfigRead) => !!webhookConfig.name?.includes("dbt"); -const toDbtCloudJob = (operation: OperationRead): DbtCloudJob => { - const { operationId } = operation; - const { executionUrl } = operation.operatorConfiguration.webhook || {}; +export const toDbtCloudJob = (operationOrCloudJob: OperationRead | DbtCloudJobInfo): DbtCloudJob => { + if ("operationId" in operationOrCloudJob) { + const { operationId } = operationOrCloudJob; + const { executionUrl } = operationOrCloudJob.operatorConfiguration.webhook || {}; - const matches = (executionUrl || "").match(/\/accounts\/([^/]+)\/jobs\/([^]+)\/run/); - if (!matches) { - throw new Error(`Cannot extract dbt cloud job params from executionUrl ${executionUrl}`); - } else { - const [, account, job] = matches; + const matches = (executionUrl || "").match(/\/accounts\/([^/]+)\/jobs\/([^]+)\/run/); + if (!matches) { + throw new Error(`Cannot extract dbt cloud job params from executionUrl ${executionUrl}`); + } else { + const [, account, job] = matches; - return { - account, - job, - operationId, - }; + return { + account, + job, + operationId, + }; + } + } else { + const { accountId, jobId, jobName } = operationOrCloudJob; + return { account: `${accountId}`, job: `${jobId}`, jobName }; } }; + const isDbtCloudJob = (operation: OperationRead): boolean => operation.operatorConfiguration.operatorType === OperatorType.webhook; +export const isSameJob = (remoteJob: DbtCloudJobInfo, savedJob: DbtCloudJob): boolean => + savedJob.account === `${remoteJob.accountId}` && savedJob.job === `${remoteJob.jobId}`; + export const useSubmitDbtCloudIntegrationConfig = () => { const { workspaceId } = useCurrentWorkspace(); const { mutateAsync: updateWorkspace } = useUpdateWorkspace(); @@ -78,35 +97,64 @@ export const useDbtIntegration = (connection: WebBackendConnectionRead) => { ); const otherOperations = [...(connection.operations?.filter((operation) => !isDbtCloudJob(operation)) || [])]; - const saveJobs = (jobs: DbtCloudJob[]) => { - // TODO dynamically use the workspace's configured dbt cloud domain when it gets returned by backend - const urlForJob = (job: DbtCloudJob) => `${dbtCloudDomain}/api/v2/accounts/${job.account}/jobs/${job.job}/run/`; - - return connectionService.update({ - connectionId: connection.connectionId, - operations: [ - ...otherOperations, - ...jobs.map((job) => ({ - workspaceId, - ...(job.operationId ? { operationId: job.operationId } : {}), - name: jobName(job), - operatorConfiguration: { - operatorType: OperatorType.webhook, - webhook: { - executionUrl: urlForJob(job), - // if `hasDbtIntegration` is true, webhookConfigId is guaranteed to exist - ...(webhookConfigId ? { webhookConfigId } : {}), - executionBody, + const { mutateAsync, isLoading } = useMutation({ + mutationFn: (jobs: DbtCloudJob[]) => { + // TODO dynamically use the workspace's configured dbt cloud domain when it gets returned by backend + const urlForJob = (job: DbtCloudJob) => `${dbtCloudDomain}/api/v2/accounts/${job.account}/jobs/${job.job}/run/`; + + return connectionService.update({ + connectionId: connection.connectionId, + operations: [ + ...otherOperations, + ...jobs.map((job) => ({ + workspaceId, + ...(job.operationId ? { operationId: job.operationId } : {}), + name: jobName(job), + operatorConfiguration: { + operatorType: OperatorType.webhook, + webhook: { + executionUrl: urlForJob(job), + // if `hasDbtIntegration` is true, webhookConfigId is guaranteed to exist + ...(webhookConfigId ? { webhookConfigId } : {}), + executionBody, + }, }, - }, - })), - ], - }); - }; + })), + ], + }); + }, + }); return { hasDbtIntegration, dbtCloudJobs, - saveJobs, + saveJobs: mutateAsync, + isSaving: isLoading, }; }; + +export const useAvailableDbtJobs = () => { + const { cloudApiUrl } = useConfig(); + const config = { apiUrl: cloudApiUrl }; + const middlewares = useDefaultRequestMiddlewares(); + const requestOptions = { config, middlewares }; + const workspace = useCurrentWorkspace(); + const { workspaceId } = workspace; + const dbtConfigId = workspace.webhookConfigs?.find((config) => config.name?.includes("dbt"))?.id; + + if (!dbtConfigId) { + throw new Error("cannot request available dbt jobs for a workspace with no dbt cloud integration configured"); + } + + const results = useQuery( + ["dbtCloud", dbtConfigId, "list"], + () => webBackendGetAvailableDbtJobsForWorkspace({ workspaceId, dbtConfigId }, requestOptions), + { + suspense: true, + } + ); + + // casting type to remove `| undefined`, since `suspense: true` will ensure the value + // is, in fact, available + return (results.data as WorkspaceGetDbtJobsResponse).availableDbtJobs; +}; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss index 98d2ce9c5f95a..5cbf2fa69897d 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss @@ -63,20 +63,20 @@ align-items: center; } -.jobListItemInputGroup { +.jobListItemIdFieldGroup { display: flex; justify-content: space-between; align-items: center; + flex-grow: 2; } -.jobListItemInput { +.jobListItemIdField { height: fit-content; margin-left: 1em; -} - -.jobListItemInputLabel { - font-size: 11px; - font-weight: 500; + background-color: colors.$grey-50; + flex-grow: 2; + padding: variables.$spacing-sm; + border-radius: variables.$border-radius-sm; } .jobListItemDelete { diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx index 37f3f88c577de..7dc068baa50e3 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.tsx @@ -1,21 +1,27 @@ import { faPlus, faXmark } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; -import { Field, Form, Formik, FieldArray, FieldProps, FormikHelpers } from "formik"; +import { Form, Formik, FieldArray, FormikHelpers } from "formik"; import { ReactNode } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Link } from "react-router-dom"; -import * as yup from "yup"; import { FormChangeTracker } from "components/common/FormChangeTracker"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; -import { Input } from "components/ui/Input"; +import { DropdownMenu } from "components/ui/DropdownMenu"; import { Text } from "components/ui/Text"; import { WebBackendConnectionRead } from "core/request/AirbyteClient"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { DbtCloudJob, useDbtIntegration } from "packages/cloud/services/dbtCloud"; +import { + DbtCloudJob, + DbtCloudJobInfo, + isSameJob, + toDbtCloudJob, + useDbtIntegration, + useAvailableDbtJobs, +} from "packages/cloud/services/dbtCloud"; import { RoutePaths } from "pages/routePaths"; import dbtLogo from "./dbt-bit_tm.svg"; @@ -26,15 +32,6 @@ interface DbtJobListValues { jobs: DbtCloudJob[]; } -const dbtCloudJobListSchema = yup.object({ - jobs: yup.array().of( - yup.object({ - account: yup.number().required().positive().integer(), - job: yup.number().required().positive().integer(), - }) - ), -}); - export const DbtCloudTransformationsCard = ({ connection }: { connection: WebBackendConnectionRead }) => { // Possible render paths: // 1) IF the workspace has no dbt cloud account linked @@ -46,18 +43,67 @@ export const DbtCloudTransformationsCard = ({ connection }: { connection: WebBac // 2.2) AND the connection has saved dbt jobs // THEN show the jobs list and the "+ Add transformation" button - const { hasDbtIntegration, saveJobs, dbtCloudJobs } = useDbtIntegration(connection); + const { hasDbtIntegration, isSaving, saveJobs, dbtCloudJobs } = useDbtIntegration(connection); + + return hasDbtIntegration ? ( + + ) : ( + + ); +}; + +const NoDbtIntegration = () => { + const { workspaceId } = useCurrentWorkspace(); + const dbtSettingsPath = `/${RoutePaths.Workspaces}/${workspaceId}/${RoutePaths.Settings}/dbt-cloud`; + return ( + + + + } + > +
+ + {linkText}, + }} + /> + +
+ + ); +}; + +interface DbtJobsFormProps { + saveJobs: (jobs: DbtCloudJob[]) => Promise; + isSaving: boolean; + dbtCloudJobs: DbtCloudJob[]; +} +const DbtJobsForm: React.FC = ({ saveJobs, isSaving, dbtCloudJobs }) => { const onSubmit = (values: DbtJobListValues, { resetForm }: FormikHelpers) => { saveJobs(values.jobs).then(() => resetForm({ values })); }; + const availableDbtJobs = useAvailableDbtJobs(); + // because we don't store names for saved jobs, just the account and job IDs needed for + // webhook operation, we have to find the display names for saved jobs by comparing IDs + // with the list of available jobs as provided by dbt Cloud. + const jobs = dbtCloudJobs.map((savedJob) => { + const { jobName } = availableDbtJobs.find((remoteJob) => isSameJob(remoteJob, savedJob)) || {}; + const { account, job } = savedJob; + + return { account, job, jobName }; + }); + return ( { - return hasDbtIntegration ? ( + initialValues={{ jobs }} + render={({ values, dirty }) => { + return (
- + {() => ( + + )} + } > - + ); }} /> - ) : ( - - - - } - > - - ); }} /> ); }; -const DbtJobsList = ({ - jobs, - remove, - isValid, - dirty, -}: { +interface DbtJobsListProps { jobs: DbtCloudJob[]; remove: (i: number) => void; - isValid: boolean; dirty: boolean; -}) => ( + isLoading: boolean; +} + +const DbtJobsList = ({ jobs, remove, dirty, isLoading }: DbtJobsListProps) => (
{jobs.length ? ( <> - {jobs.map((_, i) => ( - remove(i)} /> + {jobs.map((job, i) => ( + remove(i)} /> ))} ) : ( @@ -131,46 +171,44 @@ const DbtJobsList = ({ -
); -// TODO give feedback on validation errors (red outline and validation message) -const JobsListItem = ({ jobIndex, removeJob }: { jobIndex: number; removeJob: () => void }) => { +interface JobsListItemProps { + job: DbtCloudJob; + removeJob: () => void; +} +const JobsListItem = ({ job, removeJob }: JobsListItemProps) => { const { formatMessage } = useIntl(); + // TODO if `job.jobName` is undefined, that means we failed to match any of the + // dbt-Cloud-supplied jobs with the saved job. This means one of two things has + // happened: + // 1) the user deleted the job in dbt Cloud, and we should make them delete it from + // their webhook operations. If we have a nonempty list of other dbt Cloud jobs, + // it's definitely this. + // 2) the API call to fetch the names failed somehow (possibly with a 200 status, if there's a bug) + const title = {job.jobName || formatMessage({ id: "connection.dbtCloudJobs.job.title" })}; + return (
- + {title}
-
-
- - {({ field }: FieldProps) => ( - <> - - - - )} - +
+
+ + {formatMessage({ id: "connection.dbtCloudJobs.job.accountId" })}: {job.account} +
-
- - {({ field }: FieldProps) => ( - <> - - - - )} - +
+ + {formatMessage({ id: "connection.dbtCloudJobs.job.jobId" })}: {job.job} +