From fe0b10778951983fae3baa0592957cf34c8bd067 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 1 Feb 2024 18:47:21 -0800 Subject: [PATCH] Build js-schema files for use in schemastore --- .github/actions/publish/action.yml | 2 +- .github/workflows/pull-request-publish.yml | 11 + jsonschema/0.6.0/airbyte_protocol.json | 748 ++++++++++++++++++ jsonschema/latest/airbyte_protocol.json | 748 ++++++++++++++++++ .../bin/generate-schemastore-docker.sh | 13 + protocol-models/bin/generate-schemastore.sh | 21 + protocol-models/schemastore/package.json | 22 + protocol-models/schemastore/pnpm-lock.yaml | 162 ++++ protocol-models/schemastore/src/build.ts | 52 ++ protocol-models/schemastore/tsconfig.json | 11 + 10 files changed, 1789 insertions(+), 1 deletion(-) create mode 100644 jsonschema/0.6.0/airbyte_protocol.json create mode 100644 jsonschema/latest/airbyte_protocol.json create mode 100755 protocol-models/bin/generate-schemastore-docker.sh create mode 100755 protocol-models/bin/generate-schemastore.sh create mode 100644 protocol-models/schemastore/package.json create mode 100644 protocol-models/schemastore/pnpm-lock.yaml create mode 100644 protocol-models/schemastore/src/build.ts create mode 100644 protocol-models/schemastore/tsconfig.json diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index fc40ee5f..97fe962d 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -50,7 +50,7 @@ runs: TWINE_PASSWORD: ${{ inputs.twine_password }} TWINE_USERNAME: ${{ inputs.twine_username }} - # github runners already come with NPM and node + # js/ts: github runners already come with NPM and node - name: Generate Typescript Protocol Classes shell: bash run: protocol-models/bin/generate-typescript-classes-docker.sh diff --git a/.github/workflows/pull-request-publish.yml b/.github/workflows/pull-request-publish.yml index 81fb3971..e9e48a21 100644 --- a/.github/workflows/pull-request-publish.yml +++ b/.github/workflows/pull-request-publish.yml @@ -73,3 +73,14 @@ jobs: - name: Test npm publish run: cd protocol-models/typescript && npm publish --dry-run + + # schemastore + - name: Generate schemastore files + shell: bash + run: protocol-models/bin/generate-schemastore-docker.sh + + - name: commit new schemastore files + run: | + git add jsonschema/* + git commit -m "update schemastore files" + git push diff --git a/jsonschema/0.6.0/airbyte_protocol.json b/jsonschema/0.6.0/airbyte_protocol.json new file mode 100644 index 00000000..3d30f419 --- /dev/null +++ b/jsonschema/0.6.0/airbyte_protocol.json @@ -0,0 +1,748 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml", + "title": "AirbyteProtocol", + "type": "object", + "description": "AirbyteProtocol structs", + "version": "1.0.0", + "properties": { + "airbyte_message": { + "$ref": "#/definitions/AirbyteMessage" + }, + "configured_airbyte_catalog": { + "$ref": "#/definitions/ConfiguredAirbyteCatalog" + } + }, + "definitions": { + "AirbyteMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type" + ], + "properties": { + "type": { + "description": "Message type", + "type": "string", + "enum": [ + "RECORD", + "STATE", + "LOG", + "SPEC", + "CONNECTION_STATUS", + "CATALOG", + "TRACE", + "CONTROL" + ] + }, + "log": { + "description": "log message: any kind of logging you want the platform to know about.", + "$ref": "#/definitions/AirbyteLogMessage" + }, + "spec": { + "$ref": "#/definitions/ConnectorSpecification" + }, + "connectionStatus": { + "$ref": "#/definitions/AirbyteConnectionStatus" + }, + "catalog": { + "description": "catalog message: the catalog", + "$ref": "#/definitions/AirbyteCatalog" + }, + "record": { + "description": "record message: the record", + "$ref": "#/definitions/AirbyteRecordMessage" + }, + "state": { + "description": "schema message: the state. Must be the last message produced. The platform uses this information", + "$ref": "#/definitions/AirbyteStateMessage" + }, + "trace": { + "description": "trace message: a message to communicate information about the status and performance of a connector", + "$ref": "#/definitions/AirbyteTraceMessage" + }, + "control": { + "description": "connector config message: a message to communicate an updated configuration from a connector that should be persisted", + "$ref": "#/definitions/AirbyteControlMessage" + } + } + }, + "AirbyteRecordMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream", + "data", + "emitted_at" + ], + "properties": { + "namespace": { + "description": "namespace the data is associated with", + "type": "string" + }, + "stream": { + "description": "stream the data is associated with", + "type": "string" + }, + "data": { + "description": "record data", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "emitted_at": { + "description": "when the data was emitted from the source. epoch in millisecond.", + "type": "integer" + }, + "meta": { + "description": "Information about this record added mid-sync", + "$ref": "#/definitions/AirbyteRecordMessageMeta" + } + } + }, + "AirbyteRecordMessageMeta": { + "type": "object", + "additionalProperties": true, + "properties": { + "changes": { + "description": "Lists of changes to the content of this record which occurred during syncing", + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + } + } + } + }, + "AirbyteRecordMessageMetaChange": { + "type": "object", + "additionalProperties": true, + "required": [ + "field", + "reason", + "change" + ], + "properties": { + "field": { + "type": "string", + "description": "The field that had the change occur (required)" + }, + "change": { + "type": "string", + "description": "The type of change that occurred", + "enum": [ + "NULLED", + "TRUNCATED" + ] + }, + "reason": { + "type": "string", + "description": "The reason that the change occurred", + "enum": [ + "SOURCE_RECORD_SIZE_LIMITATION", + "DESTINATION_RECORD_SIZE_LIMITATION", + "PLATFORM_RECORD_SIZE_LIMITATION", + "SOURCE_FIELD_SIZE_LIMITATION", + "DESTINATION_FIELD_SIZE_LIMITATION", + "PLATFORM_FIELD_SIZE_LIMITATION", + "SOURCE_SERIALIZATION_ERROR", + "DESTINATION_SERIALIZATION_ERROR", + "PLATFORM_SERIALIZATION_ERROR", + "SOURCE_RETRIEVAL_ERROR" + ] + } + } + }, + "AirbyteStateMessage": { + "type": "object", + "additionalProperties": true, + "properties": { + "type": { + "$ref": "#/definitions/AirbyteStateType" + }, + "stream": { + "$ref": "#/definitions/AirbyteStreamState" + }, + "global": { + "$ref": "#/definitions/AirbyteGlobalState" + }, + "data": { + "description": "(Deprecated) the state data", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "sourceStats": { + "$ref": "#/definitions/AirbyteStateStats" + }, + "destinationStats": { + "$ref": "#/definitions/AirbyteStateStats" + } + } + }, + "AirbyteStateType": { + "type": "string", + "description": "The type of state the other fields represent. Is set to LEGACY, the state data should be read from the `data` field for backwards compatibility. If not set, assume the state object is type LEGACY. GLOBAL means that the state should be read from `global` and means that it represents the state for all the streams. It contains one shared state and individual stream states. PER_STREAM means that the state should be read from `stream`. The state present in this field correspond to the isolated state of the associated stream description.\n", + "enum": [ + "GLOBAL", + "STREAM", + "LEGACY" + ] + }, + "AirbyteStreamState": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_descriptor" + ], + "properties": { + "stream_descriptor": { + "$ref": "#/definitions/StreamDescriptor" + }, + "stream_state": { + "$ref": "#/definitions/AirbyteStateBlob" + } + } + }, + "AirbyteGlobalState": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_states" + ], + "properties": { + "shared_state": { + "$ref": "#/definitions/AirbyteStateBlob" + }, + "stream_states": { + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteStreamState" + } + } + } + }, + "StreamDescriptor": { + "type": "object", + "additionalProperties": true, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + } + }, + "AirbyteStateBlob": { + "type": "object", + "additionalProperties": true, + "description": "the state data", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "AirbyteStateStats": { + "type": "object", + "additionalProperties": true, + "properties": { + "recordCount": { + "description": "the number of records which were emitted for this state message, for this stream or global", + "type": "number" + } + } + }, + "AirbyteLogMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "level", + "message" + ], + "properties": { + "level": { + "description": "log level", + "type": "string", + "enum": [ + "FATAL", + "ERROR", + "WARN", + "INFO", + "DEBUG", + "TRACE" + ] + }, + "message": { + "description": "log message", + "type": "string" + }, + "stack_trace": { + "description": "an optional stack trace if the log message corresponds to an exception", + "type": "string" + } + } + }, + "AirbyteTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type", + "emitted_at" + ], + "properties": { + "type": { + "title": "trace type", + "description": "the type of trace message", + "type": "string", + "enum": [ + "ERROR", + "ESTIMATE", + "STREAM_STATUS", + "ANALYTICS" + ] + }, + "emitted_at": { + "description": "the time in ms that the message was emitted", + "type": "number" + }, + "error": { + "description": "error trace message: the error object", + "$ref": "#/definitions/AirbyteErrorTraceMessage" + }, + "estimate": { + "description": "Estimate trace message: a guess at how much data will be produced in this sync", + "$ref": "#/definitions/AirbyteEstimateTraceMessage" + }, + "stream_status": { + "description": "Stream status trace message: the current status of a stream within a source", + "$ref": "#/definitions/AirbyteStreamStatusTraceMessage" + }, + "analytics": { + "$ref": "#/definitions/AirbyteAnalyticsTraceMessage" + } + } + }, + "AirbyteErrorTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "message" + ], + "properties": { + "message": { + "description": "A user-friendly message that indicates the cause of the error", + "type": "string" + }, + "internal_message": { + "description": "The internal error that caused the failure", + "type": "string" + }, + "stack_trace": { + "description": "The full stack trace of the error", + "type": "string" + }, + "failure_type": { + "description": "The type of error", + "type": "string", + "enum": [ + "system_error", + "config_error" + ] + }, + "stream_descriptor": { + "description": "The stream associated with the error, if known (optional)", + "$ref": "#/definitions/StreamDescriptor" + } + } + }, + "AirbyteEstimateTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "description": "The name of the stream", + "type": "string" + }, + "type": { + "title": "estimate type", + "description": "Estimates are either per-stream (STREAM) or for the entire sync (SYNC). STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted). For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted. Sources should not emit both STREAM and SOURCE estimates within a sync.\n", + "type": "string", + "enum": [ + "STREAM", + "SYNC" + ] + }, + "namespace": { + "description": "The namespace of the stream", + "type": "string" + }, + "row_estimate": { + "description": "The estimated number of rows to be emitted by this sync for this stream", + "type": "integer" + }, + "byte_estimate": { + "description": "The estimated number of bytes to be emitted by this sync for this stream", + "type": "integer" + } + } + }, + "AirbyteStreamStatus": { + "type": "string", + "description": "The current status of a stream within the context of an executing synchronization job.\n", + "enum": [ + "STARTED", + "RUNNING", + "COMPLETE", + "INCOMPLETE" + ] + }, + "AirbyteStreamStatusTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_descriptor", + "status" + ], + "properties": { + "stream_descriptor": { + "description": "The stream associated with the status", + "$ref": "#/definitions/StreamDescriptor" + }, + "status": { + "description": "The current status of the stream", + "$ref": "#/definitions/AirbyteStreamStatus" + } + } + }, + "AirbyteAnalyticsTraceMessage": { + "type": "object", + "additionalProperties": true, + "description": "A message to communicate usage information about the connector which is not captured by regular sync analytics because it's specific to the connector internals.\nThis is useful to understand how the connector is used and how to improve it. Each message is an event with a type and an optional payload value (both of them being strings). The event types should not be dynamically generated but defined statically. The payload value is optional and can contain arbitrary strings. ", + "examples": [ + { + "type": "sql-source-used-cursor-count-indexed", + "value": "2" + }, + { + "type": "mongodb-cluster-version", + "value": "6.5" + } + ], + "required": [ + "type" + ], + "properties": { + "type": { + "description": "The event type - should be a static string. Keep in mind that all strings are shared across all connectors.", + "type": "string" + }, + "value": { + "type": "string", + "description": "The value of the event - can be an arbitrary string. In case the value is numeric, it should be converted to a string. Casting for analytics purposes can happen in the warehouse." + } + } + }, + "AirbyteControlMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type", + "emitted_at" + ], + "properties": { + "type": { + "title": "orchestrator type", + "description": "the type of orchestrator message", + "type": "string", + "enum": [ + "CONNECTOR_CONFIG" + ] + }, + "emitted_at": { + "description": "the time in ms that the message was emitted", + "type": "number" + }, + "connectorConfig": { + "description": "connector config orchestrator message: the updated config for the platform to store for this connector", + "$ref": "#/definitions/AirbyteControlConnectorConfigMessage" + } + } + }, + "AirbyteControlConnectorConfigMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "config" + ], + "properties": { + "config": { + "description": "the config items from this connector's spec to update", + "type": "object", + "additionalProperties": true + } + } + }, + "AirbyteConnectionStatus": { + "type": "object", + "description": "Airbyte connection status", + "additionalProperties": true, + "required": [ + "status" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "SUCCEEDED", + "FAILED" + ] + }, + "message": { + "type": "string" + } + } + }, + "AirbyteCatalog": { + "type": "object", + "additionalProperties": true, + "description": "Airbyte stream schema catalog", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteStream" + } + } + } + }, + "AirbyteStream": { + "type": "object", + "additionalProperties": true, + "required": [ + "name", + "json_schema", + "supported_sync_modes" + ], + "properties": { + "name": { + "type": "string", + "description": "Stream's name." + }, + "json_schema": { + "description": "Stream schema using Json Schema specs.", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "supported_sync_modes": { + "description": "List of sync modes supported by this stream.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/SyncMode" + } + }, + "source_defined_cursor": { + "description": "If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.", + "type": "boolean" + }, + "default_cursor_field": { + "description": "Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.", + "type": "array", + "items": { + "type": "string" + } + }, + "source_defined_primary_key": { + "description": "If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "namespace": { + "description": "Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.", + "type": "string" + } + } + }, + "ConfiguredAirbyteCatalog": { + "type": "object", + "additionalProperties": true, + "description": "Airbyte stream schema catalog", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "array", + "items": { + "$ref": "#/definitions/ConfiguredAirbyteStream" + } + } + } + }, + "ConfiguredAirbyteStream": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream", + "sync_mode", + "destination_sync_mode" + ], + "properties": { + "stream": { + "$ref": "#/definitions/AirbyteStream" + }, + "sync_mode": { + "$ref": "#/definitions/SyncMode", + "default": "full_refresh" + }, + "cursor_field": { + "description": "Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.", + "type": "array", + "items": { + "type": "string" + } + }, + "destination_sync_mode": { + "$ref": "#/definitions/DestinationSyncMode", + "default": "append" + }, + "primary_key": { + "description": "Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "SyncMode": { + "type": "string", + "enum": [ + "full_refresh", + "incremental" + ] + }, + "DestinationSyncMode": { + "type": "string", + "enum": [ + "append", + "overwrite", + "append_dedup" + ] + }, + "ConnectorSpecification": { + "type": "object", + "additionalProperties": true, + "description": "Specification of a connector (source/destination)", + "required": [ + "connectionSpecification" + ], + "properties": { + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "changelogUrl": { + "type": "string", + "format": "uri" + }, + "connectionSpecification": { + "description": "ConnectorDefinition specific blob. Must be a valid JSON string.", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "supportsIncremental": { + "description": "(deprecated) If the connector supports incremental mode or not.", + "type": "boolean" + }, + "supportsNormalization": { + "description": "If the connector supports normalization or not.", + "type": "boolean", + "default": false + }, + "supportsDBT": { + "description": "If the connector supports DBT or not.", + "type": "boolean", + "default": false + }, + "supported_destination_sync_modes": { + "description": "List of destination sync modes supported by the connector", + "type": "array", + "items": { + "$ref": "#/definitions/DestinationSyncMode" + } + }, + "advanced_auth": { + "description": "Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.\n - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.\n - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the\n server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical\n inputs anymore and the auth process is faster and easier to complete.", + "type": "object", + "properties": { + "auth_flow_type": { + "type": "string", + "enum": [ + "oauth2.0", + "oauth1.0" + ] + }, + "predicate_key": { + "description": "Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + "type": "array", + "items": { + "type": "string" + } + }, + "predicate_value": { + "description": "Value of the predicate_key fields for the advanced auth to be applicable.", + "type": "string" + }, + "oauth_config_specification": { + "$ref": "#/definitions/OAuthConfigSpecification" + } + } + }, + "protocol_version": { + "description": "the Airbyte Protocol version supported by the connector. Protocol versioning uses SemVer. ", + "type": "string" + } + } + }, + "OAuthConfigSpecification": { + "type": "object", + "additionalProperties": true, + "properties": { + "oauth_user_input_from_connector_config_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\n\nExamples:\n\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_output_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_server_input_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\n\nExamples:\n\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_server_output_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + } + } + } + } +} \ No newline at end of file diff --git a/jsonschema/latest/airbyte_protocol.json b/jsonschema/latest/airbyte_protocol.json new file mode 100644 index 00000000..3d30f419 --- /dev/null +++ b/jsonschema/latest/airbyte_protocol.json @@ -0,0 +1,748 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml", + "title": "AirbyteProtocol", + "type": "object", + "description": "AirbyteProtocol structs", + "version": "1.0.0", + "properties": { + "airbyte_message": { + "$ref": "#/definitions/AirbyteMessage" + }, + "configured_airbyte_catalog": { + "$ref": "#/definitions/ConfiguredAirbyteCatalog" + } + }, + "definitions": { + "AirbyteMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type" + ], + "properties": { + "type": { + "description": "Message type", + "type": "string", + "enum": [ + "RECORD", + "STATE", + "LOG", + "SPEC", + "CONNECTION_STATUS", + "CATALOG", + "TRACE", + "CONTROL" + ] + }, + "log": { + "description": "log message: any kind of logging you want the platform to know about.", + "$ref": "#/definitions/AirbyteLogMessage" + }, + "spec": { + "$ref": "#/definitions/ConnectorSpecification" + }, + "connectionStatus": { + "$ref": "#/definitions/AirbyteConnectionStatus" + }, + "catalog": { + "description": "catalog message: the catalog", + "$ref": "#/definitions/AirbyteCatalog" + }, + "record": { + "description": "record message: the record", + "$ref": "#/definitions/AirbyteRecordMessage" + }, + "state": { + "description": "schema message: the state. Must be the last message produced. The platform uses this information", + "$ref": "#/definitions/AirbyteStateMessage" + }, + "trace": { + "description": "trace message: a message to communicate information about the status and performance of a connector", + "$ref": "#/definitions/AirbyteTraceMessage" + }, + "control": { + "description": "connector config message: a message to communicate an updated configuration from a connector that should be persisted", + "$ref": "#/definitions/AirbyteControlMessage" + } + } + }, + "AirbyteRecordMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream", + "data", + "emitted_at" + ], + "properties": { + "namespace": { + "description": "namespace the data is associated with", + "type": "string" + }, + "stream": { + "description": "stream the data is associated with", + "type": "string" + }, + "data": { + "description": "record data", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "emitted_at": { + "description": "when the data was emitted from the source. epoch in millisecond.", + "type": "integer" + }, + "meta": { + "description": "Information about this record added mid-sync", + "$ref": "#/definitions/AirbyteRecordMessageMeta" + } + } + }, + "AirbyteRecordMessageMeta": { + "type": "object", + "additionalProperties": true, + "properties": { + "changes": { + "description": "Lists of changes to the content of this record which occurred during syncing", + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + } + } + } + }, + "AirbyteRecordMessageMetaChange": { + "type": "object", + "additionalProperties": true, + "required": [ + "field", + "reason", + "change" + ], + "properties": { + "field": { + "type": "string", + "description": "The field that had the change occur (required)" + }, + "change": { + "type": "string", + "description": "The type of change that occurred", + "enum": [ + "NULLED", + "TRUNCATED" + ] + }, + "reason": { + "type": "string", + "description": "The reason that the change occurred", + "enum": [ + "SOURCE_RECORD_SIZE_LIMITATION", + "DESTINATION_RECORD_SIZE_LIMITATION", + "PLATFORM_RECORD_SIZE_LIMITATION", + "SOURCE_FIELD_SIZE_LIMITATION", + "DESTINATION_FIELD_SIZE_LIMITATION", + "PLATFORM_FIELD_SIZE_LIMITATION", + "SOURCE_SERIALIZATION_ERROR", + "DESTINATION_SERIALIZATION_ERROR", + "PLATFORM_SERIALIZATION_ERROR", + "SOURCE_RETRIEVAL_ERROR" + ] + } + } + }, + "AirbyteStateMessage": { + "type": "object", + "additionalProperties": true, + "properties": { + "type": { + "$ref": "#/definitions/AirbyteStateType" + }, + "stream": { + "$ref": "#/definitions/AirbyteStreamState" + }, + "global": { + "$ref": "#/definitions/AirbyteGlobalState" + }, + "data": { + "description": "(Deprecated) the state data", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "sourceStats": { + "$ref": "#/definitions/AirbyteStateStats" + }, + "destinationStats": { + "$ref": "#/definitions/AirbyteStateStats" + } + } + }, + "AirbyteStateType": { + "type": "string", + "description": "The type of state the other fields represent. Is set to LEGACY, the state data should be read from the `data` field for backwards compatibility. If not set, assume the state object is type LEGACY. GLOBAL means that the state should be read from `global` and means that it represents the state for all the streams. It contains one shared state and individual stream states. PER_STREAM means that the state should be read from `stream`. The state present in this field correspond to the isolated state of the associated stream description.\n", + "enum": [ + "GLOBAL", + "STREAM", + "LEGACY" + ] + }, + "AirbyteStreamState": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_descriptor" + ], + "properties": { + "stream_descriptor": { + "$ref": "#/definitions/StreamDescriptor" + }, + "stream_state": { + "$ref": "#/definitions/AirbyteStateBlob" + } + } + }, + "AirbyteGlobalState": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_states" + ], + "properties": { + "shared_state": { + "$ref": "#/definitions/AirbyteStateBlob" + }, + "stream_states": { + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteStreamState" + } + } + } + }, + "StreamDescriptor": { + "type": "object", + "additionalProperties": true, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + } + }, + "AirbyteStateBlob": { + "type": "object", + "additionalProperties": true, + "description": "the state data", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "AirbyteStateStats": { + "type": "object", + "additionalProperties": true, + "properties": { + "recordCount": { + "description": "the number of records which were emitted for this state message, for this stream or global", + "type": "number" + } + } + }, + "AirbyteLogMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "level", + "message" + ], + "properties": { + "level": { + "description": "log level", + "type": "string", + "enum": [ + "FATAL", + "ERROR", + "WARN", + "INFO", + "DEBUG", + "TRACE" + ] + }, + "message": { + "description": "log message", + "type": "string" + }, + "stack_trace": { + "description": "an optional stack trace if the log message corresponds to an exception", + "type": "string" + } + } + }, + "AirbyteTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type", + "emitted_at" + ], + "properties": { + "type": { + "title": "trace type", + "description": "the type of trace message", + "type": "string", + "enum": [ + "ERROR", + "ESTIMATE", + "STREAM_STATUS", + "ANALYTICS" + ] + }, + "emitted_at": { + "description": "the time in ms that the message was emitted", + "type": "number" + }, + "error": { + "description": "error trace message: the error object", + "$ref": "#/definitions/AirbyteErrorTraceMessage" + }, + "estimate": { + "description": "Estimate trace message: a guess at how much data will be produced in this sync", + "$ref": "#/definitions/AirbyteEstimateTraceMessage" + }, + "stream_status": { + "description": "Stream status trace message: the current status of a stream within a source", + "$ref": "#/definitions/AirbyteStreamStatusTraceMessage" + }, + "analytics": { + "$ref": "#/definitions/AirbyteAnalyticsTraceMessage" + } + } + }, + "AirbyteErrorTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "message" + ], + "properties": { + "message": { + "description": "A user-friendly message that indicates the cause of the error", + "type": "string" + }, + "internal_message": { + "description": "The internal error that caused the failure", + "type": "string" + }, + "stack_trace": { + "description": "The full stack trace of the error", + "type": "string" + }, + "failure_type": { + "description": "The type of error", + "type": "string", + "enum": [ + "system_error", + "config_error" + ] + }, + "stream_descriptor": { + "description": "The stream associated with the error, if known (optional)", + "$ref": "#/definitions/StreamDescriptor" + } + } + }, + "AirbyteEstimateTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "description": "The name of the stream", + "type": "string" + }, + "type": { + "title": "estimate type", + "description": "Estimates are either per-stream (STREAM) or for the entire sync (SYNC). STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted). For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted. Sources should not emit both STREAM and SOURCE estimates within a sync.\n", + "type": "string", + "enum": [ + "STREAM", + "SYNC" + ] + }, + "namespace": { + "description": "The namespace of the stream", + "type": "string" + }, + "row_estimate": { + "description": "The estimated number of rows to be emitted by this sync for this stream", + "type": "integer" + }, + "byte_estimate": { + "description": "The estimated number of bytes to be emitted by this sync for this stream", + "type": "integer" + } + } + }, + "AirbyteStreamStatus": { + "type": "string", + "description": "The current status of a stream within the context of an executing synchronization job.\n", + "enum": [ + "STARTED", + "RUNNING", + "COMPLETE", + "INCOMPLETE" + ] + }, + "AirbyteStreamStatusTraceMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream_descriptor", + "status" + ], + "properties": { + "stream_descriptor": { + "description": "The stream associated with the status", + "$ref": "#/definitions/StreamDescriptor" + }, + "status": { + "description": "The current status of the stream", + "$ref": "#/definitions/AirbyteStreamStatus" + } + } + }, + "AirbyteAnalyticsTraceMessage": { + "type": "object", + "additionalProperties": true, + "description": "A message to communicate usage information about the connector which is not captured by regular sync analytics because it's specific to the connector internals.\nThis is useful to understand how the connector is used and how to improve it. Each message is an event with a type and an optional payload value (both of them being strings). The event types should not be dynamically generated but defined statically. The payload value is optional and can contain arbitrary strings. ", + "examples": [ + { + "type": "sql-source-used-cursor-count-indexed", + "value": "2" + }, + { + "type": "mongodb-cluster-version", + "value": "6.5" + } + ], + "required": [ + "type" + ], + "properties": { + "type": { + "description": "The event type - should be a static string. Keep in mind that all strings are shared across all connectors.", + "type": "string" + }, + "value": { + "type": "string", + "description": "The value of the event - can be an arbitrary string. In case the value is numeric, it should be converted to a string. Casting for analytics purposes can happen in the warehouse." + } + } + }, + "AirbyteControlMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "type", + "emitted_at" + ], + "properties": { + "type": { + "title": "orchestrator type", + "description": "the type of orchestrator message", + "type": "string", + "enum": [ + "CONNECTOR_CONFIG" + ] + }, + "emitted_at": { + "description": "the time in ms that the message was emitted", + "type": "number" + }, + "connectorConfig": { + "description": "connector config orchestrator message: the updated config for the platform to store for this connector", + "$ref": "#/definitions/AirbyteControlConnectorConfigMessage" + } + } + }, + "AirbyteControlConnectorConfigMessage": { + "type": "object", + "additionalProperties": true, + "required": [ + "config" + ], + "properties": { + "config": { + "description": "the config items from this connector's spec to update", + "type": "object", + "additionalProperties": true + } + } + }, + "AirbyteConnectionStatus": { + "type": "object", + "description": "Airbyte connection status", + "additionalProperties": true, + "required": [ + "status" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "SUCCEEDED", + "FAILED" + ] + }, + "message": { + "type": "string" + } + } + }, + "AirbyteCatalog": { + "type": "object", + "additionalProperties": true, + "description": "Airbyte stream schema catalog", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "array", + "items": { + "$ref": "#/definitions/AirbyteStream" + } + } + } + }, + "AirbyteStream": { + "type": "object", + "additionalProperties": true, + "required": [ + "name", + "json_schema", + "supported_sync_modes" + ], + "properties": { + "name": { + "type": "string", + "description": "Stream's name." + }, + "json_schema": { + "description": "Stream schema using Json Schema specs.", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "supported_sync_modes": { + "description": "List of sync modes supported by this stream.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/SyncMode" + } + }, + "source_defined_cursor": { + "description": "If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.", + "type": "boolean" + }, + "default_cursor_field": { + "description": "Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.", + "type": "array", + "items": { + "type": "string" + } + }, + "source_defined_primary_key": { + "description": "If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "namespace": { + "description": "Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.", + "type": "string" + } + } + }, + "ConfiguredAirbyteCatalog": { + "type": "object", + "additionalProperties": true, + "description": "Airbyte stream schema catalog", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "array", + "items": { + "$ref": "#/definitions/ConfiguredAirbyteStream" + } + } + } + }, + "ConfiguredAirbyteStream": { + "type": "object", + "additionalProperties": true, + "required": [ + "stream", + "sync_mode", + "destination_sync_mode" + ], + "properties": { + "stream": { + "$ref": "#/definitions/AirbyteStream" + }, + "sync_mode": { + "$ref": "#/definitions/SyncMode", + "default": "full_refresh" + }, + "cursor_field": { + "description": "Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.", + "type": "array", + "items": { + "type": "string" + } + }, + "destination_sync_mode": { + "$ref": "#/definitions/DestinationSyncMode", + "default": "append" + }, + "primary_key": { + "description": "Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "SyncMode": { + "type": "string", + "enum": [ + "full_refresh", + "incremental" + ] + }, + "DestinationSyncMode": { + "type": "string", + "enum": [ + "append", + "overwrite", + "append_dedup" + ] + }, + "ConnectorSpecification": { + "type": "object", + "additionalProperties": true, + "description": "Specification of a connector (source/destination)", + "required": [ + "connectionSpecification" + ], + "properties": { + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "changelogUrl": { + "type": "string", + "format": "uri" + }, + "connectionSpecification": { + "description": "ConnectorDefinition specific blob. Must be a valid JSON string.", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "supportsIncremental": { + "description": "(deprecated) If the connector supports incremental mode or not.", + "type": "boolean" + }, + "supportsNormalization": { + "description": "If the connector supports normalization or not.", + "type": "boolean", + "default": false + }, + "supportsDBT": { + "description": "If the connector supports DBT or not.", + "type": "boolean", + "default": false + }, + "supported_destination_sync_modes": { + "description": "List of destination sync modes supported by the connector", + "type": "array", + "items": { + "$ref": "#/definitions/DestinationSyncMode" + } + }, + "advanced_auth": { + "description": "Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.\n - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.\n - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the\n server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical\n inputs anymore and the auth process is faster and easier to complete.", + "type": "object", + "properties": { + "auth_flow_type": { + "type": "string", + "enum": [ + "oauth2.0", + "oauth1.0" + ] + }, + "predicate_key": { + "description": "Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + "type": "array", + "items": { + "type": "string" + } + }, + "predicate_value": { + "description": "Value of the predicate_key fields for the advanced auth to be applicable.", + "type": "string" + }, + "oauth_config_specification": { + "$ref": "#/definitions/OAuthConfigSpecification" + } + } + }, + "protocol_version": { + "description": "the Airbyte Protocol version supported by the connector. Protocol versioning uses SemVer. ", + "type": "string" + } + } + }, + "OAuthConfigSpecification": { + "type": "object", + "additionalProperties": true, + "properties": { + "oauth_user_input_from_connector_config_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\n\nExamples:\n\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_output_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_server_input_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\n\nExamples:\n\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + }, + "complete_oauth_server_output_specification": { + "description": "OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", + "type": "object", + "existingJavaType": "com.fasterxml.jackson.databind.JsonNode" + } + } + } + } +} \ No newline at end of file diff --git a/protocol-models/bin/generate-schemastore-docker.sh b/protocol-models/bin/generate-schemastore-docker.sh new file mode 100755 index 00000000..9ad296b6 --- /dev/null +++ b/protocol-models/bin/generate-schemastore-docker.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)} +ROOT_DIR_NAME=${ROOT_DIR##*/} + +[ -z "$ROOT_DIR" ] && exit 1 + +docker run --rm \ + --volume "${ROOT_DIR}:/${ROOT_DIR_NAME}" \ + --workdir "/${ROOT_DIR_NAME}" \ + node:18 "/${ROOT_DIR_NAME}/protocol-models/bin/generate-schemastore.sh" diff --git a/protocol-models/bin/generate-schemastore.sh b/protocol-models/bin/generate-schemastore.sh new file mode 100755 index 00000000..d427111d --- /dev/null +++ b/protocol-models/bin/generate-schemastore.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +git config --global --add safe.directory /airbyte-protocol + +ROOT_DIR=${ROOT_DIR:-$(git rev-parse --show-toplevel)} +[ -z "$ROOT_DIR" ] && exit 1 +cd $ROOT_DIR + +source ".env" +PATCHED_VERSION="${VERSION/.dev/"-dev"}" +echo "loaded VERSION=$PATCHED_VERSION" + +cd protocol-models/schemastore + +npm install -g pnpm + +pnpm version $PATCHED_VERSION --allow-same-version +pnpm install +pnpm run build diff --git a/protocol-models/schemastore/package.json b/protocol-models/schemastore/package.json new file mode 100644 index 00000000..e209f722 --- /dev/null +++ b/protocol-models/schemastore/package.json @@ -0,0 +1,22 @@ +{ + "name": "airbyte-protocol-schemastore", + "version": "0.6.0", + "description": "", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "node --loader ts-node/esm src/build.ts" + }, + "author": "contact@airbyte.io", + "license": "MIT", + "dependencies": { + "@types/js-yaml": "^4.0.5", + "@types/node": "^18.15.0", + "dotenv": "^16.4.1", + "js-yaml": "^4.1.0", + "ts-node": "^10.9.1", + "typescript": "^4.9.5" + } +} diff --git a/protocol-models/schemastore/pnpm-lock.yaml b/protocol-models/schemastore/pnpm-lock.yaml new file mode 100644 index 00000000..92f31b7e --- /dev/null +++ b/protocol-models/schemastore/pnpm-lock.yaml @@ -0,0 +1,162 @@ +lockfileVersion: 5.4 + +specifiers: + '@types/js-yaml': ^4.0.5 + '@types/node': ^18.15.0 + dotenv: ^16.4.1 + js-yaml: ^4.1.0 + ts-node: ^10.9.1 + typescript: ^4.9.5 + +dependencies: + '@types/js-yaml': 4.0.9 + '@types/node': 18.19.14 + dotenv: 16.4.1 + js-yaml: 4.1.0 + ts-node: 10.9.2_obavd6hot3yixo65s2mikkk65m + typescript: 4.9.5 + +packages: + + /@cspotcode/source-map-support/0.8.1: + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + dev: false + + /@jridgewell/resolve-uri/3.1.1: + resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} + engines: {node: '>=6.0.0'} + dev: false + + /@jridgewell/sourcemap-codec/1.4.15: + resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + dev: false + + /@jridgewell/trace-mapping/0.3.9: + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + dependencies: + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 + dev: false + + /@tsconfig/node10/1.0.9: + resolution: {integrity: sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==} + dev: false + + /@tsconfig/node12/1.0.11: + resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} + dev: false + + /@tsconfig/node14/1.0.3: + resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} + dev: false + + /@tsconfig/node16/1.0.4: + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + dev: false + + /@types/js-yaml/4.0.9: + resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + dev: false + + /@types/node/18.19.14: + resolution: {integrity: sha512-EnQ4Us2rmOS64nHDWr0XqAD8DsO6f3XR6lf9UIIrZQpUzPVdN/oPuEzfDWNHSyXLvoGgjuEm/sPwFGSSs35Wtg==} + dependencies: + undici-types: 5.26.5 + dev: false + + /acorn-walk/8.3.2: + resolution: {integrity: sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==} + engines: {node: '>=0.4.0'} + dev: false + + /acorn/8.11.3: + resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: false + + /arg/4.1.3: + resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} + dev: false + + /argparse/2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + + /create-require/1.1.1: + resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + dev: false + + /diff/4.0.2: + resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} + engines: {node: '>=0.3.1'} + dev: false + + /dotenv/16.4.1: + resolution: {integrity: sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==} + engines: {node: '>=12'} + dev: false + + /js-yaml/4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: false + + /make-error/1.3.6: + resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} + dev: false + + /ts-node/10.9.2_obavd6hot3yixo65s2mikkk65m: + resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + hasBin: true + peerDependencies: + '@swc/core': '>=1.2.50' + '@swc/wasm': '>=1.2.50' + '@types/node': '*' + typescript: '>=2.7' + peerDependenciesMeta: + '@swc/core': + optional: true + '@swc/wasm': + optional: true + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.9 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 18.19.14 + acorn: 8.11.3 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 4.9.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + dev: false + + /typescript/4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: false + + /undici-types/5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: false + + /v8-compile-cache-lib/3.0.1: + resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} + dev: false + + /yn/3.1.1: + resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} + engines: {node: '>=6'} + dev: false diff --git a/protocol-models/schemastore/src/build.ts b/protocol-models/schemastore/src/build.ts new file mode 100644 index 00000000..e49437d3 --- /dev/null +++ b/protocol-models/schemastore/src/build.ts @@ -0,0 +1,52 @@ +import * as path from "path"; +import * as fs from "fs"; +import { fileURLToPath } from "url"; +import * as yaml from "js-yaml"; +import dotenv from "dotenv"; + +const writeFilePretty = (p: string, content: any) => { + const json = JSON.stringify(content, null, 2); + fs.writeFileSync(p, json); + console.log(`wrote ${p}`); +}; + +const ensureDir = (p: string) => { + if (!fs.existsSync(p)) fs.mkdirSync(p); +}; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const src_dir = path.join( + __dirname, + "..", + "..", + "src", + "main", + "resources", + "airbyte_protocol" +); + +const env_file = path.join(__dirname, "..", "..", "..", ".env"); +const { parsed: config } = dotenv.config({ path: env_file }); +const version = config?.VERSION; +console.log(`writing version ${version}`); + +const src_file = path.join(src_dir, "airbyte_protocol.yaml"); +const json_object = yaml.load(fs.readFileSync(src_file, { encoding: "utf-8" })); + +const artefact_path = path.join(__dirname, "..", "..", "..", "jsonschema"); +ensureDir(artefact_path); +ensureDir(path.join(artefact_path, "latest")); +ensureDir(path.join(artefact_path, version)); + +writeFilePretty( + path.join(artefact_path, "latest", "airbyte_protocol.json"), + json_object +); + +writeFilePretty( + path.join(artefact_path, version, "airbyte_protocol.json"), + json_object +); + +process.exit(0); diff --git a/protocol-models/schemastore/tsconfig.json b/protocol-models/schemastore/tsconfig.json new file mode 100644 index 00000000..b6041035 --- /dev/null +++ b/protocol-models/schemastore/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "types": ["node"], + "moduleResolution": "node", + "module": "es2022", + "noImplicitAny": true, + "esModuleInterop": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "**/*.spec.ts"] +}