Skip to content

Commit 2d52e2f

Browse files
absorbbabsorbb
committed
console: Custom Images for connectors feature
1 parent 5d1d287 commit 2d52e2f

File tree

9 files changed

+204
-64
lines changed

9 files changed

+204
-64
lines changed

‎webapps/console/components/ConfigObjectEditor/ConfigEditor.tsx

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ export type ConfigEditorProps<T extends { id: string } = { id: string }, M = {}>
9595
//for providing custom editor component
9696
editorComponent?: EditorComponentFactory;
9797
testConnectionEnabled?: (o: any) => boolean;
98+
testButtonLabel?: string;
9899
onTest?: (o: T) => Promise<ConfigTestResult>;
99100
backTo?: string;
100101
};
@@ -326,6 +327,7 @@ const EditorComponent: React.FC<EditorComponentProps> = props => {
326327
isNew={isNew}
327328
isTouched={isTouched}
328329
hasErrors={hasErrors}
330+
testButtonLabel={props.testButtonLabel}
329331
onTest={
330332
onTest && testConnectionEnabled && testConnectionEnabled(formState?.formData || object)
331333
? async () => {

‎webapps/console/components/ConfigObjectEditor/EditorButtons.tsx

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export type EditorButtonProps<T extends { id: string } = { id: string }> = {
1717
isTouched?: boolean;
1818
hasErrors?: boolean;
1919
testStatus?: string;
20+
testButtonLabel?: string;
2021
};
2122

2223
export const EditorButtons: React.FC<EditorButtonProps> = ({
@@ -28,6 +29,7 @@ export const EditorButtons: React.FC<EditorButtonProps> = ({
2829
onSave,
2930
isTouched,
3031
hasErrors,
32+
testButtonLabel = "Test Connection",
3133
}) => {
3234
const buttonDivRef = useRef<HTMLDivElement>(null);
3335
const appConfig = useAppConfig();
@@ -93,17 +95,19 @@ export const EditorButtons: React.FC<EditorButtonProps> = ({
9395
(testStatus === "success" ? (
9496
<Popover content={"Connection test passed"} color={"lime"} trigger={"hover"}>
9597
<Button type="link" disabled={loading} size="large" onClick={doTest}>
96-
<CheckOutlined /> Test Connection
98+
<CheckOutlined />
99+
{testButtonLabel}
97100
</Button>
98101
</Popover>
99102
) : (
100103
<Button type="link" disabled={loading} size="large" onClick={doTest}>
101104
{testStatus === "pending" ? (
102105
<>
103-
<LoadingOutlined /> Test Connection
106+
<LoadingOutlined />
107+
{testButtonLabel}
104108
</>
105109
) : (
106-
"Test Connection"
110+
testButtonLabel
107111
)}
108112
</Button>
109113
))}

‎webapps/console/components/PageLayout/WorkspacePageLayout.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import React, { PropsWithChildren, ReactNode, useEffect, useState } from "react";
22
import { branding } from "../../lib/branding";
33
import { HiSelector } from "react-icons/hi";
4-
import { FaSignOutAlt, FaUserCircle } from "react-icons/fa";
4+
import { FaDocker, FaSignOutAlt, FaUserCircle } from "react-icons/fa";
55
import { FiSettings } from "react-icons/fi";
66
import { Button, Drawer, Dropdown, Menu, MenuProps } from "antd";
77
import MenuItem from "antd/lib/menu/MenuItem";
@@ -582,6 +582,7 @@ function PageHeader() {
582582
{ title: "Service Connections", path: "/services", icon: <ServerCog className="w-full h-full" /> },
583583
{ title: "Syncs", path: "/syncs", icon: <Share2 className="w-full h-full" /> },
584584
{ title: "All Logs", path: "/syncs/tasks", icon: <ScrollText className="w-full h-full" /> },
585+
{ title: "Custom Images", path: "/custom-images", icon: <FaDocker className="w-full h-full" /> },
585586
],
586587
},
587588
appConfig.ee?.available && {

‎webapps/console/components/ServicesCatalog/ServicesCatalog.tsx

Lines changed: 26 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,14 @@ import capitalize from "lodash/capitalize";
77
import { LoadingAnimation } from "../GlobalLoader/GlobalLoader";
88
import React from "react";
99
import { ErrorCard } from "../GlobalError/GlobalError";
10-
import { Button, Input, Popover } from "antd";
10+
import { Input } from "antd";
1111
import { useAppConfig, useWorkspace } from "../../lib/context";
12+
import { useConfigObjectList } from "../../lib/store";
1213

1314
function groupByType(sources: SourceType[]): Record<string, SourceType[]> {
1415
const groups: Record<string, SourceType[]> = {};
1516
const otherGroup = "other";
16-
const sortOrder = ["Datawarehouse", "Product Analytics", "CRM", "Block Storage"];
17+
const sortOrder = ["api", "database", "file", "custom image"];
1718

1819
sources.forEach(s => {
1920
if (s.packageId.endsWith("strict-encrypt") || s.packageId === "airbyte/source-file-secure") {
@@ -58,11 +59,10 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
5859
onClick,
5960
}) => {
6061
const { data, isLoading, error } = useApi<{ sources: SourceType[] }>(`/api/sources?mode=meta`);
62+
const customImages = useConfigObjectList("custom-image");
6163
const sourcesIconsLoader = useApi<{ sources: SourceType[] }>(`/api/sources?mode=icons-only`);
6264
const workspace = useWorkspace();
6365
const [filter, setFilter] = React.useState("");
64-
const [customImage, setCustomImage] = React.useState("");
65-
const [customPopupOpen, setCustomPopupOpen] = React.useState(false);
6666
const appconfig = useAppConfig();
6767
const sourcesIcons: Record<string, string> = sourcesIconsLoader.data
6868
? sourcesIconsLoader.data.sources.reduce(
@@ -79,7 +79,17 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
7979
} else if (error) {
8080
return <ErrorCard error={error} />;
8181
}
82-
const groups = groupByType(data.sources);
82+
const sources = [
83+
...data.sources,
84+
...customImages.map(c => ({
85+
id: c.package,
86+
packageId: c.package,
87+
packageType: "airbyte",
88+
meta: { name: c.name, connectorSubtype: "custom image", dockerImageTag: c.version },
89+
})),
90+
] as SourceType[];
91+
92+
const groups = groupByType(sources);
8393
return (
8494
<div className="p-6 flex flex-col flex-shrink w-full h-full overflow-y-auto">
8595
<div key={"filter"} className={"m-4"}>
@@ -123,14 +133,23 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
123133
<div
124134
key={source.id}
125135
className={`flex items-center cursor-pointer relative w-72 border border-textDisabled ${"hover:scale-105 hover:border-primary"} transition ease-in-out rounded-lg px-4 py-4 space-x-4 m-4`}
126-
onClick={() => onClick(source.packageType, source.packageId)}
136+
onClick={() =>
137+
onClick(
138+
source.packageType,
139+
source.packageId,
140+
source.meta?.connectorSubtype === "custom image" ? source.meta?.dockerImageTag : undefined
141+
)
142+
}
127143
>
128144
<div className={`${styles.icon} flex`}>{getServiceIcon(source, sourcesIcons)}</div>
129145
<div>
130146
<div>
131147
<div className={`text-xl`}>{source.meta.name}</div>
132148
</div>
133-
<div className="text-xs text-textLight">{source.packageId}</div>
149+
<div className="text-xs text-textLight">
150+
{source.packageId}
151+
{source.meta?.connectorSubtype === "custom image" ? ":" + source.meta?.dockerImageTag : ""}
152+
</div>
134153
</div>
135154
</div>
136155
);
@@ -139,52 +158,6 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
139158
</div>
140159
);
141160
})}
142-
<div key={"custom-connector"} className="">
143-
<div className="text-3xl text-textLight px-4 pb-0 pt-3">Advanced</div>
144-
<div className="flex flex-wrap">
145-
<Popover
146-
content={
147-
<div className={"flex flex-row gap-1.5"}>
148-
<Input onChange={e => setCustomImage(e.target.value)} />
149-
<Button
150-
type={"primary"}
151-
onClick={() => {
152-
const [packageId, packageVersion] = (customImage || "").trim().split(":");
153-
if (!packageId) {
154-
return;
155-
}
156-
onClick("airbyte", packageId, packageVersion);
157-
setCustomPopupOpen(false);
158-
setCustomImage("");
159-
}}
160-
>
161-
Add
162-
</Button>
163-
</div>
164-
}
165-
onOpenChange={setCustomPopupOpen}
166-
open={customPopupOpen}
167-
title="Enter docker image name"
168-
placement={"right"}
169-
trigger="click"
170-
>
171-
<div
172-
key="custom-connector"
173-
className={`flex items-center cursor-pointer relative w-72 border border-textDisabled ${"hover:scale-105 hover:border-primary"} transition ease-in-out rounded-lg px-4 py-4 space-x-4 m-4`}
174-
>
175-
<div className={`${styles.icon} flex`}>
176-
<FaDocker />
177-
</div>
178-
<div>
179-
<div>
180-
<div className={`text-xl`}>Custom connector</div>
181-
</div>
182-
<div className="text-xs text-textLight">Custom docker image</div>
183-
</div>
184-
</div>
185-
</Popover>
186-
</div>
187-
</div>
188161
</div>
189162
</div>
190163
);

‎webapps/console/lib/schema/config-objects.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
import { coreDestinationsMap } from "./destinations";
22
import { safeParseWithDate } from "../zod";
33
import { ApiError } from "../shared/errors";
4-
import { ApiKey, ConfigObjectType, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "./index";
4+
import {
5+
ApiKey,
6+
ConfigObjectType,
7+
ConnectorImageConfig,
8+
DestinationConfig,
9+
FunctionConfig,
10+
ServiceConfig,
11+
StreamConfig,
12+
} from "./index";
513
import { assertDefined, createHash, requireDefined } from "juava";
614
import { checkOrAddToIngress, isDomainAvailable } from "../server/custom-domains";
715
import { ZodType, ZodTypeDef } from "zod";
@@ -162,4 +170,7 @@ const configObjectTypes: Record<string, ConfigObjectType> = {
162170
service: {
163171
schema: ServiceConfig,
164172
},
173+
"custom-image": {
174+
schema: ConnectorImageConfig,
175+
},
165176
} as const;

‎webapps/console/lib/schema/index.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,15 @@ export const ServiceConfig = ConfigEntityBase.merge(
170170
);
171171
export type ServiceConfig = z.infer<typeof ServiceConfig>;
172172

173+
export const ConnectorImageConfig = ConfigEntityBase.merge(
174+
z.object({
175+
name: z.string(),
176+
package: z.string(),
177+
version: z.string(),
178+
})
179+
);
180+
export type ConnectorImageConfig = z.infer<typeof ConnectorImageConfig>;
181+
173182
/**
174183
* What happens to an object before it is saved to DB.
175184
*

‎webapps/console/lib/store/index.tsx

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema";
1+
import type { ConnectorImageConfig, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema";
22
import { useCallback, useEffect, useMemo, useState } from "react";
33
import { getLog, requireDefined, rpc } from "juava";
44
import { useWorkspace } from "../context";
@@ -7,7 +7,7 @@ import { z } from "zod";
77
import { ConfigurationObjectLinkDbModel, ProfileBuilderDbModel, WorkspaceDbModel } from "../../prisma/schema";
88
import { UseMutationResult } from "@tanstack/react-query/src/types";
99

10-
export const allConfigTypes = ["stream", "service", "function", "destination"] as const;
10+
export const allConfigTypes = ["stream", "service", "function", "destination", "custom-image"] as const;
1111

1212
export type ConfigType = (typeof allConfigTypes)[number];
1313

@@ -16,6 +16,7 @@ export type ConfigTypes = {
1616
service: ServiceConfig;
1717
function: FunctionConfig;
1818
destination: DestinationConfig;
19+
"custom-image": ConnectorImageConfig;
1920
};
2021

2122
export function asConfigType(type: string): ConfigType {

‎webapps/console/pages/[workspaceId]/custom-images.tsx

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import { WorkspacePageLayout } from "../../components/PageLayout/WorkspacePageLayout";
2+
import { ConfigEditor, ConfigEditorProps } from "../../components/ConfigObjectEditor/ConfigEditor";
3+
import { ConnectorImageConfig } from "../../lib/schema";
4+
import { useAppConfig, useWorkspace } from "../../lib/context";
5+
import React from "react";
6+
import { SourceType } from "../api/sources";
7+
import { ErrorCard } from "../../components/GlobalError/GlobalError";
8+
import { ServerCog } from "lucide-react";
9+
import { FaDocker } from "react-icons/fa";
10+
import { Htmlizer } from "../../components/Htmlizer/Htmlizer";
11+
import { rpc } from "juava";
12+
import { UpgradeDialog } from "../../components/Billing/UpgradeDialog";
13+
import { useBilling } from "../../components/Billing/BillingProvider";
14+
import { LoadingAnimation } from "../../components/GlobalLoader/GlobalLoader";
15+
16+
const CustomImages: React.FC<any> = () => {
17+
return (
18+
<WorkspacePageLayout>
19+
<CustomImagesList />
20+
</WorkspacePageLayout>
21+
);
22+
};
23+
24+
const CustomImagesList: React.FC<{}> = () => {
25+
const workspace = useWorkspace();
26+
const appconfig = useAppConfig();
27+
const billing = useBilling();
28+
29+
if (billing.loading) {
30+
return <LoadingAnimation />;
31+
}
32+
if (billing.enabled && billing.settings?.planId === "free") {
33+
return <UpgradeDialog featureDescription={"Custom Images"} />;
34+
}
35+
36+
if (!(appconfig.syncs.enabled || workspace.featuresEnabled.includes("syncs"))) {
37+
return (
38+
<ErrorCard
39+
title={"Feature is not enabled"}
40+
error={{ message: "'Sources Sync' feature is not enabled for current project." }}
41+
hideActions={true}
42+
/>
43+
);
44+
}
45+
46+
const config: ConfigEditorProps<ConnectorImageConfig, SourceType> = {
47+
listColumns: [
48+
{
49+
title: "Package",
50+
render: (s: ConnectorImageConfig) => <span className={"font-semibold"}>{`${s.package}:${s.version}`}</span>,
51+
},
52+
],
53+
objectType: ConnectorImageConfig,
54+
fields: {
55+
type: { constant: "custom-image" },
56+
workspaceId: { constant: workspace.id },
57+
package: {
58+
documentation: (
59+
<Htmlizer>
60+
{
61+
"Docker image name. Images can also include a registry hostname, e.g.: <code>fictional.registry.example/imagename</code>, and possibly a port number as well."
62+
}
63+
</Htmlizer>
64+
),
65+
},
66+
version: {
67+
documentation: "Docker image tag",
68+
},
69+
},
70+
noun: "custom image",
71+
type: "custom-image",
72+
explanation: "Custom connector images that can be used to setup Service connector",
73+
icon: () => <FaDocker className="w-full h-full" />,
74+
testConnectionEnabled: () => true,
75+
testButtonLabel: "Check Image",
76+
onTest: async obj => {
77+
try {
78+
const firstRes = await rpc(
79+
`/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}&force=true`
80+
);
81+
if (firstRes.ok) {
82+
return { ok: true };
83+
} else if (firstRes.error) {
84+
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${firstRes.error}` };
85+
} else {
86+
for (let i = 0; i < 60; i++) {
87+
await new Promise(resolve => setTimeout(resolve, 2000));
88+
const resp = await rpc(`/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}`);
89+
if (!resp.pending) {
90+
if (resp.error) {
91+
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${resp.error}` };
92+
} else {
93+
return { ok: true };
94+
}
95+
}
96+
}
97+
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: Timeout` };
98+
}
99+
} catch (error: any) {
100+
return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${error.message}` };
101+
}
102+
},
103+
editorTitle: (_: ConnectorImageConfig, isNew: boolean) => {
104+
const verb = isNew ? "New" : "Edit";
105+
return (
106+
<div className="flex items-center">
107+
<div className="h-12 w-12 mr-4">
108+
<FaDocker className="w-full h-full" />
109+
</div>
110+
{verb} custom image
111+
</div>
112+
);
113+
},
114+
actions: [
115+
{
116+
icon: <ServerCog className="w-full h-full" />,
117+
title: "Setup Connector",
118+
collapsed: false,
119+
link: s =>
120+
`/services?id=new&packageType=airbyte&packageId=${encodeURIComponent(s.package)}&version=${encodeURIComponent(
121+
s.version
122+
)}`,
123+
},
124+
],
125+
};
126+
return (
127+
<>
128+
<ConfigEditor {...(config as any)} />
129+
</>
130+
);
131+
};
132+
133+
export default CustomImages;

0 commit comments

Comments
 (0)