Skip to content

Commit 6767424

Browse files
avidaavida
authored
🎉 S3 source: add support for non-AWS S3 Storage (#6398)
1 parent f803481 commit 6767424

File tree

19 files changed

+287
-81
lines changed

19 files changed

+287
-81
lines changed

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/69589781-7828-43c5-9f63-8925b1c1ccc2.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
"sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2",
33
"name": "S3",
44
"dockerRepository": "airbyte/source-s3",
5-
"dockerImageTag": "0.1.4",
5+
"dockerImageTag": "0.1.5",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/s3"
77
}

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@
8585
- sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
8686
name: S3
8787
dockerRepository: airbyte/source-s3
88-
dockerImageTag: 0.1.4
88+
dockerImageTag: 0.1.5
8989
documentationUrl: https://docs.airbyte.io/integrations/sources/s3
9090
sourceType: file
9191
- sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87

‎airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ def pytest_collection_modifyitems(config, items):
100100
i += len(inner_items)
101101

102102
for items in packed_items:
103+
if not hasattr(items[0].cls, "config_key"):
104+
# Skip user defined test classes from integration_tests/ directory.
105+
continue
103106
test_configs = getattr(config.tests, items[0].cls.config_key())
104107
for test_config, item in zip(test_configs, items):
105108
default_timeout = item.get_closest_marker("default_timeout")

‎airbyte-integrations/connectors/source-s3/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ COPY source_s3 ./source_s3
1717
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
1818
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1919

20-
LABEL io.airbyte.version=0.1.4
20+
LABEL io.airbyte.version=0.1.5
2121
LABEL io.airbyte.name=airbyte/source-s3
2222

2323

‎airbyte-integrations/connectors/source-s3/acceptance-test-config.yml

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,18 @@ tests:
1111
# for Parquet format
1212
- config_path: "secrets/parquet_config.json"
1313
status: "succeed"
14+
# for custom server
15+
- config_path: "integration_tests/config_minio.json"
16+
status: "succeed"
1417
- config_path: "integration_tests/invalid_config.json"
1518
status: "failed"
1619
discovery:
17-
# for CSV format
20+
# for CSV format
1821
- config_path: "secrets/config.json"
1922
# for Parquet format
2023
- config_path: "secrets/parquet_config.json"
24+
# for custom server
25+
- config_path: "integration_tests/config_minio.json"
2126
basic_read:
2227
# for CSV format
2328
- config_path: "secrets/config.json"
@@ -29,17 +34,32 @@ tests:
2934
configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
3035
expect_records:
3136
path: "integration_tests/parquet_expected_records.txt"
32-
incremental:
37+
# for custom server
38+
- config_path: "integration_tests/config_minio.json"
39+
configured_catalog_path: "integration_tests/configured_catalog.json"
40+
# expected records contains _ab_source_file_last_modified property which
41+
# is modified all the time s3 file changed and for custom server it is
42+
# file creating date and it always new. Uncomment this line when SAT
43+
# would have ability to ignore specific fields from expected records.
44+
# expect_records:
45+
# path: "integration_tests/expected_records_custom_server.txt.txt"
46+
incremental:
3347
# for CSV format
3448
- config_path: "secrets/config.json"
3549
configured_catalog_path: "integration_tests/configured_catalog.json"
36-
cursor_paths:
50+
cursor_paths:
3751
test: ["_ab_source_file_last_modified"]
3852
future_state_path: "integration_tests/abnormal_state.json"
3953
# for Parquet format
4054
- config_path: "secrets/parquet_config.json"
4155
configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
42-
cursor_paths:
56+
cursor_paths:
57+
test: ["_ab_source_file_last_modified"]
58+
future_state_path: "integration_tests/abnormal_state.json"
59+
# for custom server
60+
- config_path: "integration_tests/config_minio.json"
61+
configured_catalog_path: "integration_tests/configured_catalog.json"
62+
cursor_paths:
4363
test: ["_ab_source_file_last_modified"]
4464
future_state_path: "integration_tests/abnormal_state.json"
4565

@@ -50,3 +70,6 @@ tests:
5070
# for Parquet format
5171
- config_path: "secrets/parquet_config.json"
5272
configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
73+
# for custom server
74+
- config_path: "integration_tests/config_minio.json"
75+
configured_catalog_path: "integration_tests/configured_catalog.json"

‎airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
#
2424

2525

26+
import shutil
27+
import tempfile
28+
from zipfile import ZipFile
29+
30+
import docker
2631
import pytest
2732

2833
pytest_plugins = ("source_acceptance_test.plugin",)
@@ -32,3 +37,22 @@
3237
def connector_setup():
3338
""" This fixture is a placeholder for external resources that acceptance test might require."""
3439
yield
40+
41+
42+
@pytest.fixture(scope="session", autouse=True)
43+
def minio_setup():
44+
client = docker.from_env()
45+
tmp_dir = tempfile.mkdtemp()
46+
with ZipFile("./integration_tests/minio_data.zip") as archive:
47+
archive.extractall(tmp_dir)
48+
49+
container = client.containers.run(
50+
"minio/minio",
51+
f"server {tmp_dir}/minio_data",
52+
network_mode="host",
53+
volumes=["/tmp:/tmp", "/var/run/docker.sock:/var/run/docker.sock"],
54+
detach=True,
55+
)
56+
yield
57+
shutil.rmtree(tmp_dir)
58+
container.stop()

‎airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"dataset": "test",
3+
"provider": {
4+
"storage": "S3",
5+
"bucket": "test-bucket",
6+
"aws_access_key_id": "123456",
7+
"aws_secret_access_key": "123456key",
8+
"path_prefix": "",
9+
"endpoint": "http://localhost:9000"
10+
},
11+
"format": {
12+
"filetype": "csv"
13+
},
14+
"path_pattern": "*.csv",
15+
"schema": "{}"
16+
}

‎airbyte-integrations/connectors/source-s3/integration_tests/expected_records_custom_server.txt

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
2+
{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
3+
{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
4+
{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
5+
{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
6+
{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
7+
{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
8+
{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
9+
{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
10+
{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
11+
{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
12+
{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
13+
{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
14+
{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
15+
{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
16+
{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
17+
{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
18+
{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
19+
{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
20+
{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
21+
{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
22+
{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
23+
{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
24+
{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
25+
{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
26+
{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
27+
{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
28+
{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
29+
{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
30+
{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
31+
{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
32+
{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
33+
{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
34+
{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
35+
{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
36+
{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
37+
{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
38+
{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
39+
{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
40+
{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
41+
{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
42+
{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
43+
{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
44+
{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
45+
{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
46+
{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
47+
{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
48+
{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
49+
{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
50+
{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
51+
{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
52+
{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
53+
{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
54+
{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
55+
{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}

‎airbyte-integrations/connectors/source-s3/integration_tests/minio_data.zip

13.1 KB
Binary file not shown.

‎airbyte-integrations/connectors/source-s3/integration_tests/spec.json

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,11 @@
116116
"title": "Columns",
117117
"description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here. Leave it empty to sync all columns.",
118118
"type": "array",
119-
"items": {
120-
"type": "string"
121-
}
119+
"items": { "type": "string" }
122120
},
123121
"batch_size": {
124122
"title": "Batch Size",
125-
"description": "Maximum number of records per batch. Batches may be smaller if there aren’t enough rows in the file. This option can help to optimize a work with memory if your data is particularly wide or failing during detection of OOM errors.",
123+
"description": "Maximum number of records per batch. Batches may be smaller if there aren\u2019t enough rows in the file. This option can help to optimize a work with memory if your data is particularly wide or failing during detection of OOM errors.",
126124
"default": 65536,
127125
"type": "integer"
128126
}
@@ -156,6 +154,22 @@
156154
"description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimise finding these in S3. This is optional but recommended if your bucket contains many folders/files.",
157155
"default": "",
158156
"type": "string"
157+
},
158+
"endpoint": {
159+
"title": "Endpoint",
160+
"description": "Endpoint to an S3 compatible service. Leave empty to use AWS.",
161+
"default": "",
162+
"type": "string"
163+
},
164+
"use_ssl": {
165+
"title": "Use Ssl",
166+
"description": "Is remote server using secure SSL/TLS connection",
167+
"type": "boolean"
168+
},
169+
"verify_ssl_cert": {
170+
"title": "Verify Ssl Cert",
171+
"description": "Allow self signed certificates",
172+
"type": "boolean"
159173
}
160174
},
161175
"required": ["bucket"]

0 commit comments

Comments
 (0)