From 98f7af58be05c2d90e2355a45c707ee32e07c3e4 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 15 Nov 2022 23:42:35 -0500 Subject: [PATCH] Add environment manager for postgres --- scripts/integration/postgres/Cargo.lock | 252 ++++++++++++++++++ scripts/integration/postgres/Cargo.toml | 16 ++ scripts/integration/postgres/README.md | 11 + .../postgres/data/docker-compose.yml | 18 ++ scripts/integration/postgres/src/core.rs | 61 +++++ scripts/integration/postgres/src/main.rs | 26 ++ scripts/integration/postgres/test.yaml | 12 + 7 files changed, 396 insertions(+) create mode 100644 scripts/integration/postgres/Cargo.lock create mode 100644 scripts/integration/postgres/Cargo.toml create mode 100644 scripts/integration/postgres/README.md create mode 100644 scripts/integration/postgres/data/docker-compose.yml create mode 100644 scripts/integration/postgres/src/core.rs create mode 100644 scripts/integration/postgres/src/main.rs create mode 100644 scripts/integration/postgres/test.yaml diff --git a/scripts/integration/postgres/Cargo.lock b/scripts/integration/postgres/Cargo.lock new file mode 100644 index 0000000000000..651226da363b9 --- /dev/null +++ b/scripts/integration/postgres/Cargo.lock @@ -0,0 +1,252 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "clap" +version = "4.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "dunce" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "os_str_bytes" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "serde" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" + +[[package]] +name = "serde_json" +version = "1.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "vector-postgres-env-manager" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "dunce", + "serde_json", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/scripts/integration/postgres/Cargo.toml b/scripts/integration/postgres/Cargo.toml new file mode 100644 index 0000000000000..e6afde474208a --- /dev/null +++ b/scripts/integration/postgres/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vector-postgres-env-manager" +version = "0.1.0" +edition = "2021" +authors = ["Vector Contributors "] +license = "MPL-2.0" +readme = "README.md" +publish = false + +[dependencies] +anyhow = "1.0.66" +clap = { version = "4.0.18", features = ["derive"] } +dunce = "1.0.3" +serde_json = "1.0.87" + +[workspace] diff --git a/scripts/integration/postgres/README.md b/scripts/integration/postgres/README.md new file mode 100644 index 0000000000000..63f0bc997bda7 --- /dev/null +++ b/scripts/integration/postgres/README.md @@ -0,0 +1,11 @@ +# vector-postgres-env-manager + +----- + +## Usage + +```text +vdev int show postgres +vdev int start postgres +vdev int stop postgres +``` diff --git a/scripts/integration/postgres/data/docker-compose.yml b/scripts/integration/postgres/data/docker-compose.yml new file mode 100644 index 0000000000000..6d046194100b2 --- /dev/null +++ b/scripts/integration/postgres/data/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + postgres: + image: docker.io/postgres:${POSTGRES_VERSION} + command: /postgres-init.sh + environment: + - POSTGRES_USER=vector + - POSTGRES_PASSWORD=vector + volumes: + - socket:/var/run/postgresql + - ../../../../tests/data/postgres-init.sh:/postgres-init.sh:ro + - ../../../../tests/data/ca:/certs:ro + +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/scripts/integration/postgres/src/core.rs b/scripts/integration/postgres/src/core.rs new file mode 100644 index 0000000000000..c75afb23c04e2 --- /dev/null +++ b/scripts/integration/postgres/src/core.rs @@ -0,0 +1,61 @@ +use anyhow::{bail, Result}; +use serde_json::Value; +use std::path::PathBuf; +use std::process::Command; +use std::thread; +use std::time::Duration; + +pub fn start(config: Value) -> Result<()> { + let mut command = compose_command(); + command.args(["up", "-d"]); + + apply_env_vars(&mut command, &config); + + let status = command.status()?; + if status.success() { + thread::sleep(Duration::from_secs(20)); + return Ok(()); + } else { + bail!("failed to execute: {}", render_command(&mut command)); + } +} + +pub fn stop(config: Value) -> Result<()> { + let mut command = compose_command(); + command.args(["down", "-t", "0"]); + + apply_env_vars(&mut command, &config); + + let status = command.status()?; + if status.success() { + return Ok(()); + } else { + bail!("failed to execute: {}", render_command(&mut command)); + } +} + +fn compose_command() -> Command { + let path = PathBuf::from_iter(["data", "docker-compose.yml"].iter()); + let compose_file = match dunce::canonicalize(&path) { + Ok(p) => p.display().to_string(), + Err(_) => path.display().to_string(), + }; + + let mut command = Command::new("docker"); + command.args(["compose", "-f", &compose_file]); + command +} + +fn apply_env_vars(command: &mut Command, config: &Value) { + if let Some(version) = config.get("version") { + command.env("POSTGRES_VERSION", version.as_str().unwrap()); + } +} + +fn render_command(command: &mut Command) -> String { + format!( + "{} {}", + command.get_program().to_str().unwrap(), + Vec::from_iter(command.get_args().map(|arg| arg.to_str().unwrap())).join(" ") + ) +} diff --git a/scripts/integration/postgres/src/main.rs b/scripts/integration/postgres/src/main.rs new file mode 100644 index 0000000000000..c063207228ab4 --- /dev/null +++ b/scripts/integration/postgres/src/main.rs @@ -0,0 +1,26 @@ +mod core; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +#[command(disable_help_subcommand = true)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + Start { json: String }, + Stop { json: String }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match &cli.command { + Commands::Start { json } => core::start(serde_json::from_str(&json)?), + Commands::Stop { json } => core::stop(serde_json::from_str(&json)?), + } +} diff --git a/scripts/integration/postgres/test.yaml b/scripts/integration/postgres/test.yaml new file mode 100644 index 0000000000000..211f6a3b5a644 --- /dev/null +++ b/scripts/integration/postgres/test.yaml @@ -0,0 +1,12 @@ +args: +- --features +- postgresql_metrics-integration-tests +- --lib +- ::postgres + +env: + PG_HOST: postgres + PG_SOCKET: /socket + +matrix: +- version: '13.1'