From 1554440dfa0faa2f8f8d16ead8e3faaff408065e Mon Sep 17 00:00:00 2001 From: Iakov Salikov Date: Thu, 16 Sep 2021 11:05:46 +0100 Subject: [PATCH 1/5] feat: path for nested fields --- airbyte-webapp/src/constants.ts | 1 + .../src/core/domain/catalog/fieldUtil.ts | 8 +++--- .../src/core/domain/catalog/models.ts | 1 + .../Connection/CatalogTree/CatalogSection.tsx | 18 +++++++------ .../Connection/CatalogTree/CatalogTree.tsx | 1 + .../views/Connection/CatalogTree/FieldRow.tsx | 27 +++++++++---------- .../Connection/CatalogTree/StreamHeader.tsx | 18 ++++++++----- 7 files changed, 43 insertions(+), 31 deletions(-) create mode 100644 airbyte-webapp/src/constants.ts diff --git a/airbyte-webapp/src/constants.ts b/airbyte-webapp/src/constants.ts new file mode 100644 index 000000000000..c0e50459880a --- /dev/null +++ b/airbyte-webapp/src/constants.ts @@ -0,0 +1 @@ +export const NESTED_FIELDS_SEPARATOR = "."; diff --git a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts index f63cb20648a9..78a28e74c03b 100644 --- a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts +++ b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts @@ -6,6 +6,7 @@ import { SourceDiscoverSchemaRead } from "./api"; import { SyncSchemaField } from "./models"; import { ConnectionNamespaceDefinition } from "../connection"; import { SOURCE_NAMESPACE_TAG } from "../connector/source"; +import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; function toInnerModel( result: SourceDiscoverSchemaRead @@ -34,7 +35,7 @@ const traverseSchemaToField = ( const traverseJsonSchemaProperties = ( jsonSchema: JSONSchema7Definition, key: string, - path: string = key, + path: string[] = [key], depth = 0 ): SyncSchemaField[] => { if (typeof jsonSchema === "boolean") { @@ -48,7 +49,7 @@ const traverseJsonSchemaProperties = ( traverseJsonSchemaProperties( schema, k, - depth === 0 ? k : `${path}.${k}`, + depth === 0 ? [k] : [...path, k], depth + 1 ) ) @@ -58,7 +59,8 @@ const traverseJsonSchemaProperties = ( return [ { cleanedName: key, - name: path, + name: path.join(NESTED_FIELDS_SEPARATOR), + path, key, fields, type: diff --git a/airbyte-webapp/src/core/domain/catalog/models.ts b/airbyte-webapp/src/core/domain/catalog/models.ts index cb99d68797dd..a618153d1a73 100644 --- a/airbyte-webapp/src/core/domain/catalog/models.ts +++ b/airbyte-webapp/src/core/domain/catalog/models.ts @@ -3,6 +3,7 @@ export type SyncSchemaField = { cleanedName: string; type: string; key: string; + path: string[]; fields?: SyncSchemaField[]; }; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx index e05db1401aaa..d647010f9db5 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx @@ -26,6 +26,7 @@ import { FieldRow } from "./FieldRow"; import { equal, naturalComparatorBy } from "utils/objects"; import { ConnectionNamespaceDefinition } from "core/domain/connection"; +import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; const flatten = ( fArr: SyncSchemaField[], @@ -98,7 +99,10 @@ const CatalogSectionInner: React.FC = ({ ); const pkPaths = useMemo( - () => new Set(config.primaryKey.map((pkPath) => pkPath.join("."))), + () => + new Set( + config.primaryKey.map((pkPath) => pkPath.join(NESTED_FIELDS_SEPARATOR)) + ), [config.primaryKey] ); @@ -163,15 +167,12 @@ const CatalogSectionInner: React.FC = ({ const flattenedFields = useMemo(() => flatten(fields), [fields]); - const primitiveFieldNames = useMemo( - () => - flattenedFields - .filter(SyncSchemaFieldObject.isPrimitive) - .map((field) => field.name), + const primitiveFields = useMemo( + () => flattenedFields.filter(SyncSchemaFieldObject.isPrimitive), [flattenedFields] ); - const selectedCursorPath = config.cursorField.join("."); + const selectedCursorPath = config.cursorField.join(NESTED_FIELDS_SEPARATOR); const configErrors = getIn(errors, `schema.streams[${streamNode.id}].config`); const hasError = configErrors && Object.keys(configErrors).length > 0; const hasChildren = fields && fields.length > 0; @@ -187,7 +188,7 @@ const CatalogSectionInner: React.FC = ({ onSelectStream={onSelectStream} onSelectSyncMode={onSelectSyncMode} isRowExpanded={isRowExpanded} - primitiveFields={primitiveFieldNames} + primitiveFields={primitiveFields} pkType={ pkRequired ? (shouldDefinePk ? "required" : "sourceDefined") : null } @@ -214,6 +215,7 @@ const CatalogSectionInner: React.FC = ({ = ({ }) => { const onUpdateStream = useCallback( (id: string, newStream: Partial) => { + console.log(newStream); const streamNode = streams.find((streamNode) => streamNode.id === id); if (streamNode) { diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx index 474826e466f3..d166ed622f26 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx @@ -1,4 +1,4 @@ -import React, { memo, useCallback } from "react"; +import React, { memo } from "react"; import styled from "styled-components"; import { Cell } from "components/SimpleTableComponents"; import { CheckBox, RadioButton } from "components"; @@ -6,6 +6,7 @@ import DataTypeCell from "./components/DataTypeCell"; interface FieldRowProps { name: string; + path: string[]; type: string; nullable?: boolean; destinationName: string; @@ -35,17 +36,12 @@ const RadiobuttonContainer = styled.div<{ depth?: number }>` padding-right: ${({ depth }) => (depth ? depth * 38 : 0)}px; `; -const FieldRowInner: React.FC = (props) => { - const { name: fieldName, onPrimaryKeyChange, onCursorChange } = props; - const handlePkChange = useCallback( - () => onPrimaryKeyChange(fieldName.split(".")), - [fieldName, onPrimaryKeyChange] - ); - - const handleCursorChange = useCallback( - () => onCursorChange(fieldName.split(".")), - [fieldName, onCursorChange] - ); +const FieldRowInner: React.FC = ({ + onPrimaryKeyChange, + onCursorChange, + path, + ...props +}) => { return ( <> @@ -61,7 +57,10 @@ const FieldRowInner: React.FC = (props) => { {props.isPrimaryKeyEnabled && ( - + onPrimaryKeyChange(path)} + /> )} @@ -69,7 +68,7 @@ const FieldRowInner: React.FC = (props) => { onCursorChange(path)} /> )} diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx index e460cfb13efd..4b6ecfa0ea6a 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx @@ -10,12 +10,14 @@ import { SyncSettingsCell } from "./components/SyncSettingsCell"; import { DestinationSyncMode, SyncMode, + SyncSchemaField, SyncSchemaStream, } from "core/domain/catalog"; import { Popout } from "components/base/Popout"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faSortDown } from "@fortawesome/free-solid-svg-icons"; import Tooltip from "./components/Tooltip"; +import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; const Arrow = styled(FontAwesomeIcon)<{ isOpen?: boolean }>` color: ${({ theme }) => theme.greyColor40}; @@ -44,7 +46,7 @@ interface StreamHeaderProps { onSelectSyncMode: (selectedMode: DropDownRow.IDataItem) => void; onSelectStream: () => void; - primitiveFields: string[]; + primitiveFields: SyncSchemaField[]; pkType: null | "required" | "sourceDefined"; onPrimaryKeyChange: (pkPath: string[][]) => void; @@ -89,8 +91,8 @@ export const StreamHeader: React.FC = ({ ); const dropdownFields = primitiveFields.map((field) => ({ - value: field.split("."), - label: field, + value: field.path, + label: field.name, })); return ( @@ -138,9 +140,13 @@ export const StreamHeader: React.FC = ({ components={PkPopupComponents} targetComponent={({ onOpen }) => (
- {primaryKey.map((k) => k.join(".")).join(", ")} + {primaryKey + .map((k) => k.join(NESTED_FIELDS_SEPARATOR)) + .join(", ")} - k.join("."))} /> + k.join(NESTED_FIELDS_SEPARATOR))} + />
)} /> @@ -159,7 +165,7 @@ export const StreamHeader: React.FC = ({ onChange={(op) => onCursorChange(op.value)} targetComponent={({ onOpen }) => (
- {stream.config.cursorField.join(".")} + {stream.config.cursorField.join(NESTED_FIELDS_SEPARATOR)}
From 56ced744b7dd41ff8209f1808f87744b547f4603 Mon Sep 17 00:00:00 2001 From: Iakov Salikov Date: Thu, 16 Sep 2021 11:08:04 +0100 Subject: [PATCH 2/5] fix: rm console.log --- airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx index 5b702183725c..a2b41a575605 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx @@ -21,7 +21,6 @@ const CatalogTree: React.FC = ({ }) => { const onUpdateStream = useCallback( (id: string, newStream: Partial) => { - console.log(newStream); const streamNode = streams.find((streamNode) => streamNode.id === id); if (streamNode) { From 34b17a6398dcf062f7718802fae153187ce3dc46 Mon Sep 17 00:00:00 2001 From: Iakov Salikov Date: Thu, 16 Sep 2021 11:20:38 +0100 Subject: [PATCH 3/5] fix: clipRule error --- .../src/views/layout/SideBar/components/ConnectionsIcon.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/views/layout/SideBar/components/ConnectionsIcon.tsx b/airbyte-webapp/src/views/layout/SideBar/components/ConnectionsIcon.tsx index e36c07780a47..6b998dff1cc7 100644 --- a/airbyte-webapp/src/views/layout/SideBar/components/ConnectionsIcon.tsx +++ b/airbyte-webapp/src/views/layout/SideBar/components/ConnectionsIcon.tsx @@ -5,8 +5,8 @@ const ConnectionsIcon = ({ }): JSX.Element => ( From b7d5e2df58e4db031646f38a4472a80ecab54ede Mon Sep 17 00:00:00 2001 From: Iakov Salikov Date: Thu, 16 Sep 2021 12:13:47 +0100 Subject: [PATCH 4/5] fix: remove field name --- airbyte-webapp/src/constants.ts | 1 - .../src/core/domain/catalog/fieldUtil.ts | 12 ++------- .../src/core/domain/catalog/models.ts | 1 - .../Connection/CatalogTree/CatalogSection.tsx | 27 +++++++++---------- .../views/Connection/CatalogTree/FieldRow.tsx | 2 +- .../Connection/CatalogTree/StreamHeader.tsx | 13 +++------ 6 files changed, 20 insertions(+), 36 deletions(-) delete mode 100644 airbyte-webapp/src/constants.ts diff --git a/airbyte-webapp/src/constants.ts b/airbyte-webapp/src/constants.ts deleted file mode 100644 index c0e50459880a..000000000000 --- a/airbyte-webapp/src/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const NESTED_FIELDS_SEPARATOR = "."; diff --git a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts index 78a28e74c03b..cb352d8b266a 100644 --- a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts +++ b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts @@ -6,7 +6,6 @@ import { SourceDiscoverSchemaRead } from "./api"; import { SyncSchemaField } from "./models"; import { ConnectionNamespaceDefinition } from "../connection"; import { SOURCE_NAMESPACE_TAG } from "../connector/source"; -import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; function toInnerModel( result: SourceDiscoverSchemaRead @@ -35,8 +34,7 @@ const traverseSchemaToField = ( const traverseJsonSchemaProperties = ( jsonSchema: JSONSchema7Definition, key: string, - path: string[] = [key], - depth = 0 + path: string[] = [] ): SyncSchemaField[] => { if (typeof jsonSchema === "boolean") { return []; @@ -46,12 +44,7 @@ const traverseJsonSchemaProperties = ( if (jsonSchema.properties) { fields = Object.entries(jsonSchema.properties) .flatMap(([k, schema]) => - traverseJsonSchemaProperties( - schema, - k, - depth === 0 ? [k] : [...path, k], - depth + 1 - ) + traverseJsonSchemaProperties(schema, k, [...path, k]) ) .flat(2); } @@ -59,7 +52,6 @@ const traverseJsonSchemaProperties = ( return [ { cleanedName: key, - name: path.join(NESTED_FIELDS_SEPARATOR), path, key, fields, diff --git a/airbyte-webapp/src/core/domain/catalog/models.ts b/airbyte-webapp/src/core/domain/catalog/models.ts index a618153d1a73..965b4f3bccc2 100644 --- a/airbyte-webapp/src/core/domain/catalog/models.ts +++ b/airbyte-webapp/src/core/domain/catalog/models.ts @@ -1,5 +1,4 @@ export type SyncSchemaField = { - name: string; cleanedName: string; type: string; key: string; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx index d647010f9db5..29e6a4989d65 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogSection.tsx @@ -26,7 +26,6 @@ import { FieldRow } from "./FieldRow"; import { equal, naturalComparatorBy } from "utils/objects"; import { ConnectionNamespaceDefinition } from "core/domain/connection"; -import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; const flatten = ( fArr: SyncSchemaField[], @@ -98,14 +97,6 @@ const CatalogSectionInner: React.FC = ({ [config, updateStreamWithConfig] ); - const pkPaths = useMemo( - () => - new Set( - config.primaryKey.map((pkPath) => pkPath.join(NESTED_FIELDS_SEPARATOR)) - ), - [config.primaryKey] - ); - const onPkSelect = useCallback( (pkPath: string[]) => { let newPrimaryKey: string[][]; @@ -172,11 +163,19 @@ const CatalogSectionInner: React.FC = ({ [flattenedFields] ); - const selectedCursorPath = config.cursorField.join(NESTED_FIELDS_SEPARATOR); const configErrors = getIn(errors, `schema.streams[${streamNode.id}].config`); const hasError = configErrors && Object.keys(configErrors).length > 0; const hasChildren = fields && fields.length > 0; + const isCursor = (field: SyncSchemaField): boolean => + equal(config.cursorField, field.path); + + const isPrimaryKey = (field: SyncSchemaField): boolean => { + const existIndex = config.primaryKey.findIndex((p) => equal(p, field.path)); + + return existIndex !== -1; + }; + return (
@@ -212,15 +211,15 @@ const CatalogSectionInner: React.FC = ({ {flattenedFields.map((field) => ( - + = ({ return ( <> - + {props.name} diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx index 4b6ecfa0ea6a..2eafe3a2945e 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx @@ -17,7 +17,6 @@ import { Popout } from "components/base/Popout"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faSortDown } from "@fortawesome/free-solid-svg-icons"; import Tooltip from "./components/Tooltip"; -import { NESTED_FIELDS_SEPARATOR } from "../../../constants"; const Arrow = styled(FontAwesomeIcon)<{ isOpen?: boolean }>` color: ${({ theme }) => theme.greyColor40}; @@ -92,7 +91,7 @@ export const StreamHeader: React.FC = ({ const dropdownFields = primitiveFields.map((field) => ({ value: field.path, - label: field.name, + label: field.path.join("."), })); return ( @@ -140,13 +139,9 @@ export const StreamHeader: React.FC = ({ components={PkPopupComponents} targetComponent={({ onOpen }) => (
- {primaryKey - .map((k) => k.join(NESTED_FIELDS_SEPARATOR)) - .join(", ")} + {primaryKey.map((k) => k.join(".")).join(", ")} - k.join(NESTED_FIELDS_SEPARATOR))} - /> + k.join("."))} />
)} /> @@ -165,7 +160,7 @@ export const StreamHeader: React.FC = ({ onChange={(op) => onCursorChange(op.value)} targetComponent={({ onOpen }) => (
- {stream.config.cursorField.join(NESTED_FIELDS_SEPARATOR)} + {stream.config.cursorField.join(".")}
From 83ce120b7f841ef58c73a9cf02e573729b2d3408 Mon Sep 17 00:00:00 2001 From: Iakov Salikov Date: Thu, 16 Sep 2021 13:26:30 +0100 Subject: [PATCH 5/5] fix: name in FieldRow --- airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx index 76f64fe79438..d166ed622f26 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/FieldRow.tsx @@ -45,7 +45,7 @@ const FieldRowInner: React.FC = ({ return ( <> - + {props.name}