Skip to content

Commit 3d429b4

Browse files
cirdesmarcosmarxmoctavia-squidington-iii
authored and
nataly
committed
🎉 New Destination: Typesense (#18349)
* Initial boilerplate * 🎉 New Destination: Typesense * remove .java-version * fix doc * add typesense to dest def * add release stage * add requirement to main * auto-bump connector version * add changelog Co-authored-by: marcosmarxm <[email protected]> Co-authored-by: Marcos Marx <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 8472db7 commit 3d429b4

File tree

17 files changed

+568
-1
lines changed

17 files changed

+568
-1
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,12 @@
329329
documentationUrl: https://docs.airbyte.com/integrations/destinations/tidb
330330
icon: tidb.svg
331331
releaseStage: alpha
332+
- name: Typesense
333+
destinationDefinitionId: 36be8dc6-9851-49af-b776-9d4c30e4ab6a
334+
dockerRepository: airbyte/destination-typesense
335+
dockerImageTag: 0.1.0
336+
documentationUrl: https://docs.airbyte.com/integrations/destinations/typesense
337+
releaseStage: alpha
332338
- name: YugabyteDB
333339
destinationDefinitionId: 2300fdcf-a532-419f-9f24-a014336e7966
334340
dockerRepository: airbyte/destination-yugabytedb

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6145,6 +6145,51 @@
61456145
supported_destination_sync_modes:
61466146
- "overwrite"
61476147
- "append"
6148+
- dockerImage: "airbyte/destination-typesense:0.1.0"
6149+
spec:
6150+
documentationUrl: "https://docs.airbyte.com/integrations/destinations/typesense"
6151+
connectionSpecification:
6152+
$schema: "http://json-schema.org/draft-07/schema#"
6153+
title: "Destination Typesense"
6154+
type: "object"
6155+
required:
6156+
- "api_key"
6157+
- "host"
6158+
additionalProperties: false
6159+
properties:
6160+
api_key:
6161+
title: "API Key"
6162+
type: "string"
6163+
description: "Typesense API Key"
6164+
order: 0
6165+
host:
6166+
title: "Host"
6167+
type: "string"
6168+
description: "Hostname of the Typesense instance without protocol."
6169+
order: 1
6170+
port:
6171+
title: "Port"
6172+
type: "string"
6173+
description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\
6174+
\ is 443"
6175+
order: 2
6176+
protocol:
6177+
title: "Protocol"
6178+
type: "string"
6179+
description: "Protocol of the Typesense instance. Ex: http or https. Default\
6180+
\ is https"
6181+
order: 3
6182+
batch_size:
6183+
title: "Batch size"
6184+
type: "string"
6185+
description: "How many documents should be imported together. Default 1000"
6186+
order: 4
6187+
supportsIncremental: true
6188+
supportsNormalization: false
6189+
supportsDBT: false
6190+
supported_destination_sync_modes:
6191+
- "overwrite"
6192+
- "append"
61486193
- dockerImage: "airbyte/destination-yugabytedb:0.1.0"
61496194
spec:
61506195
documentationUrl: "https://docs.airbyte.io/integrations/destinations/yugabytedb"

‎airbyte-integrations/connectors/destination-typesense/.dockerignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
*
2+
!Dockerfile
3+
!main.py
4+
!destination_typesense
5+
!setup.py

‎airbyte-integrations/connectors/destination-typesense/Dockerfile

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
FROM python:3.9.11-alpine3.15 as base
2+
3+
# build and load all requirements
4+
FROM base as builder
5+
WORKDIR /airbyte/integration_code
6+
7+
# upgrade pip to the latest version
8+
RUN apk --no-cache upgrade \
9+
&& pip install --upgrade pip \
10+
&& apk --no-cache add tzdata build-base
11+
12+
13+
COPY setup.py ./
14+
# install necessary packages to a temporary folder
15+
RUN pip install --prefix=/install .
16+
17+
# build a clean environment
18+
FROM base
19+
WORKDIR /airbyte/integration_code
20+
21+
# copy all loaded and built libraries to a pure basic image
22+
COPY --from=builder /install /usr/local
23+
# add default timezone settings
24+
COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
25+
RUN echo "Etc/UTC" > /etc/timezone
26+
27+
# bash is installed for more convenient debugging.
28+
RUN apk --no-cache add bash
29+
30+
# copy payload code only
31+
COPY main.py ./
32+
COPY destination_typesense ./destination_typesense
33+
34+
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
35+
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
36+
37+
LABEL io.airbyte.version=0.1.0
38+
LABEL io.airbyte.name=airbyte/destination-typesense

‎airbyte-integrations/connectors/destination-typesense/README.md

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Typesense Destination
2+
3+
This is the repository for the Typesense destination connector, written in Python.
4+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/typesense).
5+
6+
## Local development
7+
8+
### Prerequisites
9+
**To iterate on this connector, make sure to complete this prerequisites section.**
10+
11+
#### Minimum Python version required `= 3.7.0`
12+
13+
#### Build & Activate Virtual Environment and install dependencies
14+
From this connector directory, create a virtual environment:
15+
```
16+
python -m venv .venv
17+
```
18+
19+
This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
20+
development environment of choice. To activate it from the terminal, run:
21+
```
22+
source .venv/bin/activate
23+
pip install -r requirements.txt
24+
```
25+
If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
26+
27+
Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
28+
used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
29+
If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
30+
should work as you expect.
31+
32+
#### Building via Gradle
33+
From the Airbyte repository root, run:
34+
```
35+
./gradlew :airbyte-integrations:connectors:destination-typesense:build
36+
```
37+
38+
#### Create credentials
39+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/typesense)
40+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_typesense/spec.json` file.
41+
Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information.
42+
See `integration_tests/sample_config.json` for a sample config file.
43+
44+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination typesense test creds`
45+
and place them into `secrets/config.json`.
46+
47+
### Locally running the connector
48+
```
49+
python main.py spec
50+
python main.py check --config secrets/config.json
51+
python main.py discover --config secrets/config.json
52+
python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
53+
```
54+
55+
### Locally running the connector docker image
56+
57+
#### Build
58+
First, make sure you build the latest Docker image:
59+
```
60+
docker build . -t airbyte/destination-typesense:dev
61+
```
62+
63+
You can also build the connector image via Gradle:
64+
```
65+
./gradlew :airbyte-integrations:connectors:destination-typesense:airbyteDocker
66+
```
67+
When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
68+
the Dockerfile.
69+
70+
#### Run
71+
Then run any of the connector commands as follows:
72+
```
73+
docker run --rm airbyte/destination-typesense:dev spec
74+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-typesense:dev check --config /secrets/config.json
75+
# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages
76+
cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-typesense:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
77+
```
78+
## Testing
79+
Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
80+
First install test dependencies into your virtual environment:
81+
```
82+
pip install .[tests]
83+
```
84+
### Unit Tests
85+
To run unit tests locally, from the connector directory run:
86+
```
87+
python -m pytest unit_tests
88+
```
89+
90+
### Integration Tests
91+
There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector).
92+
#### Custom Integration tests
93+
Place custom tests inside `integration_tests/` folder, then, from the connector root, run
94+
```
95+
python -m pytest integration_tests
96+
```
97+
#### Acceptance Tests
98+
Coming soon:
99+
100+
### Using gradle to run tests
101+
All commands should be run from airbyte project root.
102+
To run unit tests:
103+
```
104+
./gradlew :airbyte-integrations:connectors:destination-typesense:unitTest
105+
```
106+
To run acceptance and custom integration tests:
107+
```
108+
./gradlew :airbyte-integrations:connectors:destination-typesense:integrationTest
109+
```
110+
111+
## Dependency Management
112+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
113+
We split dependencies between two groups, dependencies that are:
114+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
115+
* required for the testing need to go to `TEST_REQUIREMENTS` list
116+
117+
### Publishing a new version of the connector
118+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
119+
1. Make sure your changes are passing unit and integration tests.
120+
1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)).
121+
1. Create a Pull Request.
122+
1. Pat yourself on the back for being an awesome contributor.
123+
1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.

‎airbyte-integrations/connectors/destination-typesense/build.gradle

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
plugins {
2+
id 'airbyte-python'
3+
id 'airbyte-docker'
4+
}
5+
6+
airbytePython {
7+
moduleDirectory 'destination_typesense'
8+
}

‎airbyte-integrations/connectors/destination-typesense/destination_typesense/__init__.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#
2+
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
6+
from .destination import DestinationTypesense
7+
8+
__all__ = ["DestinationTypesense"]

‎airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#
2+
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
6+
from logging import Logger
7+
from typing import Any, Iterable, Mapping
8+
9+
from airbyte_cdk.destinations import Destination
10+
from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type
11+
from destination_typesense.writer import TypesenseWriter
12+
from typesense import Client
13+
14+
15+
def get_client(config: Mapping[str, Any]) -> Client:
16+
api_key = config.get("api_key")
17+
host = config.get("host")
18+
port = config.get("port") or "8108"
19+
protocol = config.get("protocol") or "https"
20+
21+
client = Client({"api_key": api_key, "nodes": [{"host": host, "port": port, "protocol": protocol}], "connection_timeout_seconds": 2})
22+
23+
return client
24+
25+
26+
class DestinationTypesense(Destination):
27+
def write(
28+
self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage]
29+
) -> Iterable[AirbyteMessage]:
30+
client = get_client(config=config)
31+
32+
for configured_stream in configured_catalog.streams:
33+
steam_name = configured_stream.stream.name
34+
if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite:
35+
try:
36+
client.collections[steam_name].delete()
37+
except Exception:
38+
pass
39+
client.collections.create({"name": steam_name, "fields": [{"name": ".*", "type": "auto"}]})
40+
41+
writer = TypesenseWriter(client, steam_name, config.get("batch_size"))
42+
for message in input_messages:
43+
if message.type == Type.STATE:
44+
writer.flush()
45+
yield message
46+
elif message.type == Type.RECORD:
47+
writer.queue_write_operation(message.record.data)
48+
else:
49+
continue
50+
writer.flush()
51+
52+
def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
53+
try:
54+
client = get_client(config=config)
55+
client.collections.create({"name": "_airbyte", "fields": [{"name": "title", "type": "string"}]})
56+
client.collections["_airbyte"].documents.create({"id": "1", "title": "The Hunger Games"})
57+
client.collections["_airbyte"].documents["1"].retrieve()
58+
client.collections["_airbyte"].delete()
59+
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
60+
except Exception as e:
61+
return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}")

‎airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"documentationUrl": "https://docs.airbyte.com/integrations/destinations/typesense",
3+
"supported_destination_sync_modes": ["overwrite", "append"],
4+
"supportsIncremental": true,
5+
"supportsDBT": false,
6+
"supportsNormalization": false,
7+
"connectionSpecification": {
8+
"$schema": "http://json-schema.org/draft-07/schema#",
9+
"title": "Destination Typesense",
10+
"type": "object",
11+
"required": ["api_key", "host"],
12+
"additionalProperties": false,
13+
"properties": {
14+
"api_key": {
15+
"title": "API Key",
16+
"type": "string",
17+
"description": "Typesense API Key",
18+
"order": 0
19+
},
20+
"host": {
21+
"title": "Host",
22+
"type": "string",
23+
"description": "Hostname of the Typesense instance without protocol.",
24+
"order": 1
25+
},
26+
"port": {
27+
"title": "Port",
28+
"type": "string",
29+
"description": "Port of the Typesense instance. Ex: 8108, 80, 443. Default is 443",
30+
"order": 2
31+
},
32+
"protocol": {
33+
"title": "Protocol",
34+
"type": "string",
35+
"description": "Protocol of the Typesense instance. Ex: http or https. Default is https",
36+
"order": 3
37+
},
38+
"batch_size": {
39+
"title": "Batch size",
40+
"type": "string",
41+
"description": "How many documents should be imported together. Default 1000",
42+
"order": 4
43+
}
44+
}
45+
}
46+
}

‎airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#
2+
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
from collections.abc import Mapping
6+
from logging import getLogger
7+
from uuid import uuid4
8+
9+
from typesense import Client
10+
11+
logger = getLogger("airbyte")
12+
13+
14+
class TypesenseWriter:
15+
write_buffer = []
16+
17+
def __init__(self, client: Client, steam_name: str, batch_size: int = 1000):
18+
self.client = client
19+
self.steam_name = steam_name
20+
self.batch_size = batch_size
21+
22+
def queue_write_operation(self, data: Mapping):
23+
random_key = str(uuid4())
24+
data_with_id = data if "id" in data else {**data, "id": random_key}
25+
self.write_buffer.append(data_with_id)
26+
if len(self.write_buffer) == self.batch_size:
27+
self.flush()
28+
29+
def flush(self):
30+
buffer_size = len(self.write_buffer)
31+
if buffer_size == 0:
32+
return
33+
logger.info(f"flushing {buffer_size} records")
34+
self.client.collections[self.steam_name].documents.import_(self.write_buffer)
35+
self.write_buffer.clear()

0 commit comments

Comments
 (0)