From 4b6684e53a306fae314536591e774bbb3a889814 Mon Sep 17 00:00:00 2001 From: nameton1 Date: Tue, 7 May 2024 12:37:42 +0200 Subject: [PATCH 01/10] Adding docs on Influxio --- docs/integrate/etl/influxdb.md | 191 +++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 docs/integrate/etl/influxdb.md diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md new file mode 100644 index 00000000..53f3d253 --- /dev/null +++ b/docs/integrate/etl/influxdb.md @@ -0,0 +1,191 @@ +(influxdb-import)= +# Import data from InfluxDB + +In this quick tutorial we use [influxio](https://github.com/daq-tools/influxio) +to import data from InfluxDB into CrateDB. It can be used both as a standalone +program, and as a library. + +:::{note} +Please note that influxio is alpha-quality software, and a work in progress. +Contributions of all kinds are very welcome, in order to make it more solid. +Breaking changes should be expected until a 1.0 release, so version pinning is +recommended, especially when you use it as a library. +::: + +(setup-influxio)= +## Setup + +`influxio` can be installed with pip: + +:::{code} console +pip install influxio +::: + +You will also need the [cratedb-toolkit](https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io#installation) +to load the data into your CrateDB instance. It can be installed with: + +:::{code} console +pip install --upgrade 'cratedb-toolkit[all]' +::: + +(setup-influxdb)= +### InfluxDB setup + +The following InfluxDB setup will be used in this tutorial: + +:::{code} console +influx setup \ + --username user1 \ + --password 49jk8FQB$]1 \ + --token token123 \ + --org example \ + --bucket testdrive \ + --force +::: + +With the following sample data loaded in InfluxDB: + +:::{code} console +influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" +influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=28.2,humidity=88.7,windspeed=4.7 1588549600" +influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=27.9,humidity=91.6,windspeed=3.2 1588736000" +influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=29.1,humidity=88.1,windspeed=2.4 1588922400" +influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=28.6,humidity=93.4,windspeed=2.9 1589108800" +::: + +(export-data)= +### Export + +Now you can export the data into CrateDB: + +:::{code} console +export CRATEDB_SQLALCHEMY_URL=crate://crate@localhost:4200/testdrive/demo +ctk load table influxdb2://example:token123@localhost:8086/testdrive/demo +::: + +And verify that data is indeed in your CrateDB cluster: + +:::{code} console +crash --command "SELECT * FROM testdrive.demo;" +::: + +(cmd-use)= +## Command line use + +Below are various examples of usage. + +(cmd-help)= +### Help + +:::{code} console +influxio --help +influxio info +influxio copy --help +::: + +(cmd-import)= +### Import + +:::{code} console + # From test data to API. + # Choose one of dummy, mixed, dateindex, wide. + influxio copy \ + "testdata://dateindex/" \ + "http://example:token@localhost:8086/testdrive/demo" + + # With selected amount of rows. + influxio copy \ + "testdata://dateindex/?rows=42" \ + "http://example:token@localhost:8086/testdrive/demo" + + # With selected amount of rows and columns (only supported by certain test data sources). + influxio copy \ + "testdata://wide/?rows=42&columns=42" \ + "http://example:token@localhost:8086/testdrive/demo" + + # From line protocol file to InfluxDB API. + influxio copy \ + "file://tests/testdata/basic.lp" \ + "http://example:token@localhost:8086/testdrive/demo" + + # From line protocol file to InfluxDB API. + influxio copy \ + "https://github.com/influxdata/influxdb2-sample-data/raw/master/air-sensor-data/air-sensor-data.lp" \ + "http://example:token@localhost:8086/testdrive/demo" + + # From line protocol file to any database supported by SQLAlchemy. + influxio copy \ + "file://export.lp" \ + "sqlite://export.sqlite?table=export" +::: + +(cmd-export-from-api)= +### Export from api + +:::{code} console + # From API to database file. + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "sqlite:///export.sqlite?table=demo" + + # From API to database server. + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "crate://crate@localhost:4200/testdrive?table=demo" + + # From API to line protocol file. + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "file://export.lp" + + # From API to line protocol on stdout. + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "file://-?format=lp" +::: + +(cmd-export-from-data-dir)= +### Export from data directory + +:::{code} console + # From InfluxDB data directory to line protocol file. + influxio copy \ + "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ + "file://export.lp" + + # From InfluxDB data directory to line protocol file, compressed with gzip. + influxio copy \ + "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ + "file://export.lp.gz" + + # From InfluxDB data directory to line protocol on stdout. + influxio copy \ + "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ + ""file://-?format=lp" +::: + +(cmd-oci)= +### OCI + +OCI images are available on the GitHub Container Registry (GHCR). +In order to run them on Podman or Docker, invoke: + +:::{code} console +docker run --rm --network=host ghcr.io/daq-tools/influxio \ + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "stdout://export.lp" +::: + +If you want to work with files on your filesystem, you will need to either +mount the working directory into the container using the --volume option, +or use the --interactive option to consume STDIN, like: + +:::{code} console +docker run --rm --volume=$(pwd):/data ghcr.io/daq-tools/influxio \ + influxio copy "file:///data/export.lp" "sqlite:///data/export.sqlite?table=export" + +cat export.lp | \ +docker run --rm --interactive --network=host ghcr.io/daq-tools/influxio \ + influxio copy "stdin://?format=lp" "crate://crate@localhost:4200/testdrive/export" +::: From b2796f253cddddc423d5c751827490a6f32eab26 Mon Sep 17 00:00:00 2001 From: Matej Matkuliak <53749658+matkuliak@users.noreply.github.com> Date: Tue, 7 May 2024 19:12:22 +0200 Subject: [PATCH 02/10] Apply suggestions from code review Co-authored-by: Andreas Motl --- docs/integrate/etl/influxdb.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index 53f3d253..ff2dde33 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -1,16 +1,8 @@ (influxdb-import)= # Import data from InfluxDB -In this quick tutorial we use [influxio](https://github.com/daq-tools/influxio) -to import data from InfluxDB into CrateDB. It can be used both as a standalone -program, and as a library. - -:::{note} -Please note that influxio is alpha-quality software, and a work in progress. -Contributions of all kinds are very welcome, in order to make it more solid. -Breaking changes should be expected until a 1.0 release, so version pinning is -recommended, especially when you use it as a library. -::: +In this quick tutorial we use the InfluxDB I/O subsystem of CrateDB Toolkit +to import data from InfluxDB into CrateDB. (setup-influxio)= ## Setup From d27ec7e15026f79c4b73e6f1ec82f458859019f8 Mon Sep 17 00:00:00 2001 From: matkuliak Date: Tue, 7 May 2024 19:20:23 +0200 Subject: [PATCH 03/10] added to index and toc --- docs/integrate/etl/index.md | 4 ++++ docs/integrate/etl/influxdb.md | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/integrate/etl/index.md b/docs/integrate/etl/index.md index 75a45ba2..a205b26a 100644 --- a/docs/integrate/etl/index.md +++ b/docs/integrate/etl/index.md @@ -72,6 +72,9 @@ Tutorials and resources about configuring the managed variants, Astro and CrateD - {ref}`integrate-mongodb` +## InfluxDB + +- {ref}`integrate-influxdb` ## MySQL @@ -136,6 +139,7 @@ A demo project which uses SSIS and ODBC to read and write data from CrateDB: ```{toctree} :hidden: +influxdb mongodb mysql ``` diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index ff2dde33..9ae0c12b 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -1,4 +1,5 @@ -(influxdb-import)= +(integrate-influxdb)= +(import-influxdb)= # Import data from InfluxDB In this quick tutorial we use the InfluxDB I/O subsystem of CrateDB Toolkit From b1afa5486bb2aa1775170ef9ed17539b07f7965e Mon Sep 17 00:00:00 2001 From: matkuliak Date: Tue, 7 May 2024 19:41:24 +0200 Subject: [PATCH 04/10] fix outdated link --- docs/integrate/bi/powerbi-desktop.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrate/bi/powerbi-desktop.rst b/docs/integrate/bi/powerbi-desktop.rst index 7169c353..6e6c27de 100644 --- a/docs/integrate/bi/powerbi-desktop.rst +++ b/docs/integrate/bi/powerbi-desktop.rst @@ -192,6 +192,6 @@ The pie chart will be updated automatically, and will produce the following: .. _World Economic Outlook survey: https://www.imf.org/en/Publications/WEO .. _Power BI Desktop: https://powerbi.microsoft.com/en-us/desktop/ .. _PostgreSQL ODBC driver: https://odbc.postgresql.org/ -.. _downloads section: https://www.postgresql.org/ftp/odbc/versions/msi/ +.. _downloads section: https://www.postgresql.org/ftp/odbc/versions.old/msi/ .. _raw data: https://www.imf.org/en/Publications/WEO/weo-database/2017/April/download-entire-database .. _preprocessed archive: https://cratedb.com/wp-content/uploads/2018/11/copy_from_population_data.zip From 1e6012559c23f2a96037ccdbdb8cd5e0043fd0ee Mon Sep 17 00:00:00 2001 From: matkuliak Date: Tue, 7 May 2024 19:43:25 +0200 Subject: [PATCH 05/10] sort links alphabetically --- docs/integrate/etl/index.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/integrate/etl/index.md b/docs/integrate/etl/index.md index a205b26a..f778bd96 100644 --- a/docs/integrate/etl/index.md +++ b/docs/integrate/etl/index.md @@ -62,20 +62,18 @@ Tutorials and resources about configuring the managed variants, Astro and CrateD - [Tutorial: Replicating data to CrateDB with Debezium and Kafka] - [Webinar: How to replicate data from other databases to CrateDB with Debezium and Kafka] +## InfluxDB + +- {ref}`integrate-influxdb` ## Kestra - [Setting up data pipelines with CrateDB and Kestra] - ## MongoDB - {ref}`integrate-mongodb` -## InfluxDB - -- {ref}`integrate-influxdb` - ## MySQL - {ref}`integrate-mysql` From fa9a5c3cefaf08cfc6d1c51cbf25345588f81897 Mon Sep 17 00:00:00 2001 From: matkuliak Date: Tue, 7 May 2024 20:21:10 +0200 Subject: [PATCH 06/10] address suggestions --- docs/integrate/etl/influxdb.md | 157 +++++---------------------------- 1 file changed, 22 insertions(+), 135 deletions(-) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index 9ae0c12b..ea35bbbd 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -5,26 +5,26 @@ In this quick tutorial we use the InfluxDB I/O subsystem of CrateDB Toolkit to import data from InfluxDB into CrateDB. -(setup-influxio)= -## Setup - -`influxio` can be installed with pip: - -:::{code} console -pip install influxio -::: +(integrate-influxdb-quickstart)= +## Quickstart -You will also need the [cratedb-toolkit](https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io#installation) -to load the data into your CrateDB instance. It can be installed with: +There are multiple ways to get and use the this tool, to avoid +unnecessary installations we will use Docker to run the services. +**Docker is needed for this:** -:::{code} console -pip install --upgrade 'cratedb-toolkit[all]' -::: +docker run --rm --network=host ghcr.io/daq-tools/influxio \ + influxio copy \ + "http://example:token@localhost:8086/testdrive/demo" \ + "crate://crate@localhost:4200/testdrive/demo" (setup-influxdb)= ### InfluxDB setup -The following InfluxDB setup will be used in this tutorial: +Following is an example configuration of InfluxDB and some sample data +should you need it. **Prerequisite for these to work is a running +instance of InfluxDB.** + +Initial InfluxDB configuration: :::{code} console influx setup \ @@ -36,7 +36,7 @@ influx setup \ --force ::: -With the following sample data loaded in InfluxDB: +Write sample data to InfluxDB: :::{code} console influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" @@ -49,7 +49,8 @@ influx write --bucket=testdrive --org=example --precision=s "demo,region=amazona (export-data)= ### Export -Now you can export the data into CrateDB: +Now you can export the data into CrateDB. **Prerequisite for these to work +is a running instance of CrateDB.** :::{code} console export CRATEDB_SQLALCHEMY_URL=crate://crate@localhost:4200/testdrive/demo @@ -62,123 +63,9 @@ And verify that data is indeed in your CrateDB cluster: crash --command "SELECT * FROM testdrive.demo;" ::: -(cmd-use)= -## Command line use - -Below are various examples of usage. - -(cmd-help)= -### Help - -:::{code} console -influxio --help -influxio info -influxio copy --help -::: - -(cmd-import)= -### Import - -:::{code} console - # From test data to API. - # Choose one of dummy, mixed, dateindex, wide. - influxio copy \ - "testdata://dateindex/" \ - "http://example:token@localhost:8086/testdrive/demo" - - # With selected amount of rows. - influxio copy \ - "testdata://dateindex/?rows=42" \ - "http://example:token@localhost:8086/testdrive/demo" - - # With selected amount of rows and columns (only supported by certain test data sources). - influxio copy \ - "testdata://wide/?rows=42&columns=42" \ - "http://example:token@localhost:8086/testdrive/demo" - - # From line protocol file to InfluxDB API. - influxio copy \ - "file://tests/testdata/basic.lp" \ - "http://example:token@localhost:8086/testdrive/demo" - - # From line protocol file to InfluxDB API. - influxio copy \ - "https://github.com/influxdata/influxdb2-sample-data/raw/master/air-sensor-data/air-sensor-data.lp" \ - "http://example:token@localhost:8086/testdrive/demo" - - # From line protocol file to any database supported by SQLAlchemy. - influxio copy \ - "file://export.lp" \ - "sqlite://export.sqlite?table=export" -::: - -(cmd-export-from-api)= -### Export from api - -:::{code} console - # From API to database file. - influxio copy \ - "http://example:token@localhost:8086/testdrive/demo" \ - "sqlite:///export.sqlite?table=demo" - - # From API to database server. - influxio copy \ - "http://example:token@localhost:8086/testdrive/demo" \ - "crate://crate@localhost:4200/testdrive?table=demo" - - # From API to line protocol file. - influxio copy \ - "http://example:token@localhost:8086/testdrive/demo" \ - "file://export.lp" - - # From API to line protocol on stdout. - influxio copy \ - "http://example:token@localhost:8086/testdrive/demo" \ - "file://-?format=lp" -::: - -(cmd-export-from-data-dir)= -### Export from data directory - -:::{code} console - # From InfluxDB data directory to line protocol file. - influxio copy \ - "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ - "file://export.lp" - - # From InfluxDB data directory to line protocol file, compressed with gzip. - influxio copy \ - "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ - "file://export.lp.gz" - - # From InfluxDB data directory to line protocol on stdout. - influxio copy \ - "file:///path/to/influxdb/engine?bucket-id=372d1908eab801a6&measurement=demo" \ - ""file://-?format=lp" -::: - -(cmd-oci)= -### OCI - -OCI images are available on the GitHub Container Registry (GHCR). -In order to run them on Podman or Docker, invoke: - -:::{code} console -docker run --rm --network=host ghcr.io/daq-tools/influxio \ - influxio copy \ - "http://example:token@localhost:8086/testdrive/demo" \ - "stdout://export.lp" -::: - -If you want to work with files on your filesystem, you will need to either -mount the working directory into the container using the --volume option, -or use the --interactive option to consume STDIN, like: - -:::{code} console -docker run --rm --volume=$(pwd):/data ghcr.io/daq-tools/influxio \ - influxio copy "file:///data/export.lp" "sqlite:///data/export.sqlite?table=export" +(cratedb-toolkit)= +## Setup -cat export.lp | \ -docker run --rm --interactive --network=host ghcr.io/daq-tools/influxio \ - influxio copy "stdin://?format=lp" "crate://crate@localhost:4200/testdrive/export" -::: +There are many more ways to get and use the CrateDB Toolkit. To see all +that's possible with it, please visit the +[cratedb-toolkit repository](https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io) From 550940054656c0c6cef821123b31c5328c6a6ed1 Mon Sep 17 00:00:00 2001 From: matkuliak Date: Tue, 7 May 2024 20:26:16 +0200 Subject: [PATCH 07/10] forgot code wrap --- docs/integrate/etl/influxdb.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index ea35bbbd..47a6a7a8 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -12,10 +12,12 @@ There are multiple ways to get and use the this tool, to avoid unnecessary installations we will use Docker to run the services. **Docker is needed for this:** +:::{code} console docker run --rm --network=host ghcr.io/daq-tools/influxio \ influxio copy \ "http://example:token@localhost:8086/testdrive/demo" \ "crate://crate@localhost:4200/testdrive/demo" +::: (setup-influxdb)= ### InfluxDB setup From 27a85475b9350bbfc01ccd313c926325edd34a36 Mon Sep 17 00:00:00 2001 From: Matej Matkuliak <53749658+matkuliak@users.noreply.github.com> Date: Tue, 7 May 2024 20:50:35 +0200 Subject: [PATCH 08/10] Update docs/integrate/etl/influxdb.md Co-authored-by: Andreas Motl --- docs/integrate/etl/influxdb.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index 47a6a7a8..7e478ae1 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -65,9 +65,15 @@ And verify that data is indeed in your CrateDB cluster: crash --command "SELECT * FROM testdrive.demo;" ::: -(cratedb-toolkit)= -## Setup +## More information -There are many more ways to get and use the CrateDB Toolkit. To see all -that's possible with it, please visit the -[cratedb-toolkit repository](https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io) +There are many more ways to apply the I/O subsystem of CrateDB Toolkit as pipeline +elements in your daily data operations routines. Please visit the [CrateDB Toolkit I/O +Documentation], to learn more about what's possible. + +The InfluxDB I/O subsystem is based on the [influxio] package. Please also check its +documentation to learn about more of its capabilities, supporting you when working +with InfluxDB. + +[influxio]: https://github.com/daq-tools/influxio +[CrateDB Toolkit I/O Documentation]: https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io From 3e59a16650dce311903ca701272e3ce3b7d75e34 Mon Sep 17 00:00:00 2001 From: Matej Matkuliak <53749658+matkuliak@users.noreply.github.com> Date: Wed, 8 May 2024 13:21:25 +0200 Subject: [PATCH 09/10] Update docs/integrate/etl/influxdb.md Co-authored-by: Andreas Motl --- docs/integrate/etl/influxdb.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index 7e478ae1..a477e59f 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -75,5 +75,5 @@ The InfluxDB I/O subsystem is based on the [influxio] package. Please also check documentation to learn about more of its capabilities, supporting you when working with InfluxDB. -[influxio]: https://github.com/daq-tools/influxio -[CrateDB Toolkit I/O Documentation]: https://github.com/crate-workbench/cratedb-toolkit/tree/main/cratedb_toolkit/io +[influxio]: https://influxio.readthedocs.io/ +[CrateDB Toolkit I/O Documentation]: https://cratedb-toolkit.readthedocs.io/io/influxdb/loader.html From 156ba155700aa383fdec08bcc2e90453e8f904f1 Mon Sep 17 00:00:00 2001 From: Matej Matkuliak <53749658+matkuliak@users.noreply.github.com> Date: Thu, 9 May 2024 00:50:55 +0200 Subject: [PATCH 10/10] dockerize rest of the setup --- docs/integrate/etl/influxdb.md | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/docs/integrate/etl/influxdb.md b/docs/integrate/etl/influxdb.md index a477e59f..010f8b31 100644 --- a/docs/integrate/etl/influxdb.md +++ b/docs/integrate/etl/influxdb.md @@ -29,23 +29,26 @@ instance of InfluxDB.** Initial InfluxDB configuration: :::{code} console -influx setup \ - --username user1 \ - --password 49jk8FQB$]1 \ - --token token123 \ - --org example \ - --bucket testdrive \ - --force +docker run --rm -it --publish=8086:8086 --name influxdb\ + --env=DOCKER_INFLUXDB_INIT_MODE=setup \ + --env=DOCKER_INFLUXDB_INIT_USERNAME=user1 \ + --env=DOCKER_INFLUXDB_INIT_PASSWORD=secret1234 \ + --env=DOCKER_INFLUXDB_INIT_ORG=example \ + --env=DOCKER_INFLUXDB_INIT_BUCKET=testdrive \ + --env=DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=token \ + --volume="$PWD/var/lib/influxdb2:/var/lib/influxdb2" \ + influxdb:latest ::: Write sample data to InfluxDB: :::{code} console -influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" -influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=28.2,humidity=88.7,windspeed=4.7 1588549600" -influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=27.9,humidity=91.6,windspeed=3.2 1588736000" -influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=29.1,humidity=88.1,windspeed=2.4 1588922400" -influx write --bucket=testdrive --org=example --precision=s "demo,region=amazonas temperature=28.6,humidity=93.4,windspeed=2.9 1589108800" +docker exec influxdb influx write --bucket=testdrive --org=example --precision=s --token=token "demo,region=amazonas temperature=27.4,humidity=92.3,windspeed=4.5 1588363200" +docker exec influxdb influx write --bucket=testdrive --org=example --precision=s --token=token "demo,region=amazonas temperature=28.2,humidity=88.7,windspeed=4.7 1588549600" +docker exec influxdb influx write --bucket=testdrive --org=example --precision=s --token=token "demo,region=amazonas temperature=27.9,humidity=91.6,windspeed=3.2 1588736000" +docker exec influxdb influx write --bucket=testdrive --org=example --precision=s --token=token "demo,region=amazonas temperature=29.1,humidity=88.1,windspeed=2.4 1588922400" +docker exec influxdb influx write --bucket=testdrive --org=example --precision=s --token=token "demo,region=amazonas temperature=28.6,humidity=93.4,windspeed=2.9 1589108800" + ::: (export-data)= @@ -54,9 +57,16 @@ influx write --bucket=testdrive --org=example --precision=s "demo,region=amazona Now you can export the data into CrateDB. **Prerequisite for these to work is a running instance of CrateDB.** +First, create these aliases so the next part is a bit easier: + +::::{code} console +alias crash="docker run --rm -it ghcr.io/crate-workbench/cratedb-toolkit:latest crash" +alias ctk="docker run --rm -it ghcr.io/crate-workbench/cratedb-toolkit:latest ctk" +::: + :::{code} console export CRATEDB_SQLALCHEMY_URL=crate://crate@localhost:4200/testdrive/demo -ctk load table influxdb2://example:token123@localhost:8086/testdrive/demo +ctk load table influxdb2://example:token@localhost:8086/testdrive/demo ::: And verify that data is indeed in your CrateDB cluster: @@ -67,13 +77,13 @@ crash --command "SELECT * FROM testdrive.demo;" ## More information -There are many more ways to apply the I/O subsystem of CrateDB Toolkit as pipeline -elements in your daily data operations routines. Please visit the [CrateDB Toolkit I/O -Documentation], to learn more about what's possible. +There are many more ways to apply the I/O subsystem of CrateDB Toolkit as +pipeline elements in your daily data operations routines. Please visit the +[CrateDB Toolkit I/O Documentation], to learn more about what's possible. -The InfluxDB I/O subsystem is based on the [influxio] package. Please also check its -documentation to learn about more of its capabilities, supporting you when working -with InfluxDB. +The InfluxDB I/O subsystem is based on the [influxio] package. Please also +check its documentation to learn about more of its capabilities, supporting +you when working with InfluxDB. [influxio]: https://influxio.readthedocs.io/ [CrateDB Toolkit I/O Documentation]: https://cratedb-toolkit.readthedocs.io/io/influxdb/loader.html