diff --git a/dataline-config-persistence/src/main/java/io/dataline/config/persistence/ConfigPersistenceImpl.java b/dataline-config-persistence/src/main/java/io/dataline/config/persistence/ConfigPersistenceImpl.java index c4b80f2b4ed02..314b92bed4af7 100644 --- a/dataline-config-persistence/src/main/java/io/dataline/config/persistence/ConfigPersistenceImpl.java +++ b/dataline-config-persistence/src/main/java/io/dataline/config/persistence/ConfigPersistenceImpl.java @@ -114,24 +114,32 @@ private String getFilename(String id) { private ConfigSchema standardConfigTypeToConfigSchema( PersistenceConfigType persistenceConfigType) { switch (persistenceConfigType) { - case SOURCE_CONNECTION_CONFIGURATION: - return ConfigSchema.SOURCE_CONNECTION_CONFIGURATION; + case STANDARD_WORKSPACE: + return ConfigSchema.STANDARD_WORKSPACE; + case STANDARD_SOURCE: + return ConfigSchema.STANDARD_SOURCE; + case SOURCE_CONNECTION_SPECIFICATION: + return ConfigSchema.SOURCE_CONNECTION_SPECIFICATION; + case SOURCE_CONNECTION_IMPLEMENTATION: + return ConfigSchema.SOURCE_CONNECTION_IMPLEMENTATION; + case STANDARD_DESTINATION: + return ConfigSchema.STANDARD_DESTINATION; + case DESTINATION_CONNECTION_SPECIFICATION: + return ConfigSchema.DESTINATION_CONNECTION_SPECIFICATION; + case DESTINATION_CONNECTION_IMPLEMENTATION: + return ConfigSchema.DESTINATION_CONNECTION_IMPLEMENTATION; case STANDARD_CONNECTION_STATUS: return ConfigSchema.STANDARD_CONNECTION_STATUS; case STANDARD_DISCOVERY_OUTPUT: return ConfigSchema.STANDARD_DISCOVERY_OUTPUT; - case DESTINATION_CONNECTION_CONFIGURATION: - return ConfigSchema.DESTINATION_CONNECTION_CONFIGURATION; - case STANDARD_SYNC_CONFIGURATION: - return ConfigSchema.STANDARD_SYNC_CONFIGURATION; + case STANDARD_SYNC: + return ConfigSchema.STANDARD_SYNC; case STANDARD_SYNC_SUMMARY: return ConfigSchema.STANDARD_SYNC_SUMMARY; - case STANDARD_SYNC_STATE: - return ConfigSchema.STANDARD_SYNC_STATE; - case STATE: - return ConfigSchema.STATE; case STANDARD_SYNC_SCHEDULE: return ConfigSchema.STANDARD_SYNC_SCHEDULE; + case STATE: + return ConfigSchema.STATE; default: throw new RuntimeException( String.format( diff --git a/dataline-config-persistence/src/main/java/io/dataline/config/persistence/PersistenceConfigType.java b/dataline-config-persistence/src/main/java/io/dataline/config/persistence/PersistenceConfigType.java index 16034f9f0e6d0..d08a3184a7b81 100644 --- a/dataline-config-persistence/src/main/java/io/dataline/config/persistence/PersistenceConfigType.java +++ b/dataline-config-persistence/src/main/java/io/dataline/config/persistence/PersistenceConfigType.java @@ -1,13 +1,28 @@ package io.dataline.config.persistence; public enum PersistenceConfigType { - SOURCE_CONNECTION_CONFIGURATION, + // workspace + STANDARD_WORKSPACE, + + // source + STANDARD_SOURCE, + SOURCE_CONNECTION_SPECIFICATION, + SOURCE_CONNECTION_IMPLEMENTATION, + + // destination + STANDARD_DESTINATION, + DESTINATION_CONNECTION_SPECIFICATION, + DESTINATION_CONNECTION_IMPLEMENTATION, + + // test connection STANDARD_CONNECTION_STATUS, + + // discover schema STANDARD_DISCOVERY_OUTPUT, - DESTINATION_CONNECTION_CONFIGURATION, - STANDARD_SYNC_CONFIGURATION, + + // sync + STANDARD_SYNC, STANDARD_SYNC_SUMMARY, - STANDARD_SYNC_STATE, + STANDARD_SYNC_SCHEDULE, STATE, - STANDARD_SYNC_SCHEDULE } diff --git a/dataline-config/src/main/java/io/dataline/config/ConfigSchema.java b/dataline-config/src/main/java/io/dataline/config/ConfigSchema.java index a786703bf7253..a50ae7fc9d859 100644 --- a/dataline-config/src/main/java/io/dataline/config/ConfigSchema.java +++ b/dataline-config/src/main/java/io/dataline/config/ConfigSchema.java @@ -1,13 +1,28 @@ package io.dataline.config; public enum ConfigSchema { - SOURCE_CONNECTION_CONFIGURATION("SourceConnectionConfiguration.json"), + // workspace + STANDARD_WORKSPACE("StandardWorkspace.json"), + + // source + STANDARD_SOURCE("StandardSource.json"), + SOURCE_CONNECTION_SPECIFICATION("SourceConnectionSpecification.json"), + SOURCE_CONNECTION_IMPLEMENTATION("SourceConnectionImplementation.json"), + + // destination + STANDARD_DESTINATION("StandardDestination.json"), + DESTINATION_CONNECTION_SPECIFICATION("DestinationConnectionSpecification.json"), + DESTINATION_CONNECTION_IMPLEMENTATION("DestinationConnectionImplementation.json"), + + // test connection STANDARD_CONNECTION_STATUS("StandardConnectionStatus.json"), + + // discover schema STANDARD_DISCOVERY_OUTPUT("StandardDiscoveryOutput.json"), - DESTINATION_CONNECTION_CONFIGURATION("DestinationConnectionConfiguration.json"), - STANDARD_SYNC_CONFIGURATION("StandardSyncConfiguration.json"), + + // sync + STANDARD_SYNC("StandardSync.json"), STANDARD_SYNC_SUMMARY("StandardSyncSummary.json"), - STANDARD_SYNC_STATE("StandardSyncState.json"), STANDARD_SYNC_SCHEDULE("StandardSyncSchedule.json"), STATE("State.json"); diff --git a/dataline-config/src/main/resources/json/DestinationConnectionConfiguration.json b/dataline-config/src/main/resources/json/DestinationConnectionConfiguration.json deleted file mode 100644 index f130732331cbe..0000000000000 --- a/dataline-config/src/main/resources/json/DestinationConnectionConfiguration.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "DestinationConnectionConfiguration", - "title": "DestinationConnectionConfiguration", - "description": "information required for connection to a destination.", - "type": "object", - "required": ["destinationSpecificationId", "configuration"], - "additionalProperties": false, - "properties": { - "destinationSpecificationId": { - "type": "string", - "format": "uuid" - }, - "configuration": { - } - } -} diff --git a/dataline-config/src/main/resources/json/DestinationConnectionImplementation.json b/dataline-config/src/main/resources/json/DestinationConnectionImplementation.json new file mode 100644 index 0000000000000..8b511cef501c4 --- /dev/null +++ b/dataline-config/src/main/resources/json/DestinationConnectionImplementation.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/DestinationConnectionImplementation.json", + "title": "DestinationConnectionImplementation", + "description": "information required for connection to a destination.", + "type": "object", + "required": ["destinationSpecificationId", "destinationImplementationId", "configuration"], + "additionalProperties": false, + "properties": { + "destinationSpecificationId": { + "type": "string", + "format": "uuid" + }, + "destinationImplementationId": { + "type": "string", + "format": "uuid" + }, + "configuration": { + "description": "type any. information here varies by integration. this data is validated against a specification" + } + } +} diff --git a/dataline-config/src/main/resources/json/DestinationConnectionSpecification.json b/dataline-config/src/main/resources/json/DestinationConnectionSpecification.json new file mode 100644 index 0000000000000..daa5ab86a26ae --- /dev/null +++ b/dataline-config/src/main/resources/json/DestinationConnectionSpecification.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/DestinationSpecification.json", + "title": "DestinationConnectionSpecification", + "description": "specification for how to configure a connection to a destination", + "type": "object", + "required": ["destinationId", "destinationSpecificationId", "specification"], + "additionalProperties": false, + "properties": { + "destinationId": { + "type": "string", + "format": "uuid" + }, + "destinationSpecificationId": { + "type": "string", + "format": "uuid" + }, + "specification": { + "description": "type any. information here varies by integration" + } + } +} diff --git a/dataline-config/src/main/resources/json/SourceConnectionConfiguration.json b/dataline-config/src/main/resources/json/SourceConnectionConfiguration.json deleted file mode 100644 index 3f3e29097cb0b..0000000000000 --- a/dataline-config/src/main/resources/json/SourceConnectionConfiguration.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "SourceConnectionConfiguration", - "title": "SourceConnectionConfiguration", - "description": "information required for connection to a destination.", "type": "object", - "required": ["sourceSpecificationId", "configuration"], - "additionalProperties": false, - "properties": { - "sourceSpecificationId": { - "type": "string", - "format": "uuid" - }, - "configuration": { - } - } -} diff --git a/dataline-config/src/main/resources/json/SourceConnectionImplementation.json b/dataline-config/src/main/resources/json/SourceConnectionImplementation.json new file mode 100644 index 0000000000000..40786974e519c --- /dev/null +++ b/dataline-config/src/main/resources/json/SourceConnectionImplementation.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/SourceConnectionConfiguration.json", + "title": "SourceConnectionConfiguration", + "description": "information required for connection to a destination.", "type": "object", + "required": ["sourceSpecificationId", "sourceImplementationId", "configuration"], + "additionalProperties": false, + "properties": { + "sourceSpecificationId": { + "type": "string", + "format": "uuid" + }, + "sourceImplementationId": { + "type": "string", + "format": "uuid" + }, + "configuration": { + "description": "type any. information here varies by integration. this data is validated against a specification" + } + } +} diff --git a/dataline-config/src/main/resources/json/SourceConnectionSpecification.json b/dataline-config/src/main/resources/json/SourceConnectionSpecification.json new file mode 100644 index 0000000000000..5f7a194481c36 --- /dev/null +++ b/dataline-config/src/main/resources/json/SourceConnectionSpecification.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/SourceSpecification.json", + "title": "SourceConnectionSpecification", + "description": "specification for how to configure a connection to a source", + "type": "object", + "required": ["sourceId", "sourceSpecificationId", "specification"], + "additionalProperties": false, + "properties": { + "sourceId": { + "type": "string", + "format": "uuid" + }, + "sourceSpecificationId": { + "type": "string", + "format": "uuid" + }, + "specification": { + "description": "type any. information here varies by integration" + } + } +} diff --git a/dataline-config/src/main/resources/json/StandardConnectionStatus.json b/dataline-config/src/main/resources/json/StandardConnectionStatus.json index 901c0946488ca..7c48fb9276779 100644 --- a/dataline-config/src/main/resources/json/StandardConnectionStatus.json +++ b/dataline-config/src/main/resources/json/StandardConnectionStatus.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "StandardConnectionStatus", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardConnectionStatus.json", "title": "StandardConnectionStatus", "description": "describes the result of a 'test connection' action.", "type": "object", diff --git a/dataline-config/src/main/resources/json/StandardDataSchema.json b/dataline-config/src/main/resources/json/StandardDataSchema.json index e46842e3a19fc..d56e87d181868 100644 --- a/dataline-config/src/main/resources/json/StandardDataSchema.json +++ b/dataline-config/src/main/resources/json/StandardDataSchema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "StandardDataSchema", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardDataSchema.json", "title": "StandardDataSchema", "type": "object", "definitions": { diff --git a/dataline-config/src/main/resources/json/StandardDestination.json b/dataline-config/src/main/resources/json/StandardDestination.json new file mode 100644 index 0000000000000..76d4669890f9d --- /dev/null +++ b/dataline-config/src/main/resources/json/StandardDestination.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/Destination.json", + "title": "Source", + "description": "describes a destination", + "type": "object", + "required": [ + "destinationId", + "name" + ], + "additionalProperties": false, + "properties": { + "destinationId": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + } + } +} diff --git a/dataline-config/src/main/resources/json/StandardDiscoveryOutput.json b/dataline-config/src/main/resources/json/StandardDiscoveryOutput.json index 98928a4327774..c1fc531673a72 100644 --- a/dataline-config/src/main/resources/json/StandardDiscoveryOutput.json +++ b/dataline-config/src/main/resources/json/StandardDiscoveryOutput.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "StandardDiscoveryOutput", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardDiscoveryOutput.json", "title": "StandardDiscoveryOutput", "description": "describes the standard output for any discovery run.", "type": "object", diff --git a/dataline-config/src/main/resources/json/StandardSource.json b/dataline-config/src/main/resources/json/StandardSource.json new file mode 100644 index 0000000000000..8b67ade27e900 --- /dev/null +++ b/dataline-config/src/main/resources/json/StandardSource.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/Source.json", + "title": "Source", + "description": "describes a source", + "type": "object", + "required": [ + "sourceId", + "name" + ], + "additionalProperties": false, + "properties": { + "sourceId": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + } + } +} diff --git a/dataline-config/src/main/resources/json/StandardSyncConfiguration.json b/dataline-config/src/main/resources/json/StandardSync.json similarity index 75% rename from dataline-config/src/main/resources/json/StandardSyncConfiguration.json rename to dataline-config/src/main/resources/json/StandardSync.json index 8360fddfdae7d..521a7af2834e7 100644 --- a/dataline-config/src/main/resources/json/StandardSyncConfiguration.json +++ b/dataline-config/src/main/resources/json/StandardSync.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "StandardSyncConfiguration", - "title": "StandardSyncConfiguration", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardSyncConfiguration.json", + "title": "StandardSync", "description": "configuration required for sync for ALL taps", "type": "object", "required": ["syncMode", "schema"], diff --git a/dataline-config/src/main/resources/json/StandardSyncSchedule.json b/dataline-config/src/main/resources/json/StandardSyncSchedule.json index d65236aa6588f..327dcd1a41b29 100644 --- a/dataline-config/src/main/resources/json/StandardSyncSchedule.json +++ b/dataline-config/src/main/resources/json/StandardSyncSchedule.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://dataline.io/docs/models/StandardScheduleConfiguration.json", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardScheduleConfiguration.json", "title": "StandardScheduleConfiguration", "type": "object", "required": ["timeUnit", "units"], diff --git a/dataline-config/src/main/resources/json/StandardSyncSummary.json b/dataline-config/src/main/resources/json/StandardSyncSummary.json index 7d8438d4a19e8..bbad91fbea1e9 100644 --- a/dataline-config/src/main/resources/json/StandardSyncSummary.json +++ b/dataline-config/src/main/resources/json/StandardSyncSummary.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "StandardSyncSummary", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardSyncSummary.json", "title": "StandardSyncSummary", "description": "standard information output by ALL taps for a sync step (our version of state.json)", "type": "object", diff --git a/dataline-config/src/main/resources/json/StandardWorkspace.json b/dataline-config/src/main/resources/json/StandardWorkspace.json new file mode 100644 index 0000000000000..7185c2b2cde7b --- /dev/null +++ b/dataline-config/src/main/resources/json/StandardWorkspace.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/StandardWorkspaceConfiguration.json", + "title": "StandardWorkspace", + "description": "workspace configuration", + "type": "object", + "required": ["workspaceId", "name", "slug", "initialSetupComplete"], + "additionalProperties": false, + "properties": { + "workspaceId": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "initialSetupComplete": { + "type": "boolean" + }, + "anonymousDataCollection": { + "type": "boolean" + }, + "news": { + "type": "boolean" + }, + "securityUpdates": { + "type": "boolean" + } + } +} diff --git a/dataline-config/src/main/resources/json/State.json b/dataline-config/src/main/resources/json/State.json index 9e90d59d98862..d5c17b94ef35e 100644 --- a/dataline-config/src/main/resources/json/State.json +++ b/dataline-config/src/main/resources/json/State.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "DestinationConnectionConfiguration", - "title": "DestinationConnectionConfiguration", + "$id": "https://github.com/datalineio/dataline/blob/master/dataline-config/src/main/resources/json/State.json", + "title": "State", "description": "information output by the connection.", "type": "object", "required": ["connectionId", "state"], @@ -12,6 +12,7 @@ "format": "uuid" }, "state": { + "description": "type any. information here varies by integration" } } }