diff --git a/.github/workflows/_test_int.yml b/.github/workflows/_test_int.yml index 502ab85a0..6b1daebc1 100644 --- a/.github/workflows/_test_int.yml +++ b/.github/workflows/_test_int.yml @@ -37,7 +37,12 @@ jobs: uses: supercharge/mongodb-github-action@1.8.0 with: mongodb-version: ${{ inputs.mongodb }} - mongodb-replica-set: test-rs + mongodb-username: root + mongodb-password: root + # FIXME: Currently we cannot configure this action to use authentication together with replica sets as mentioned here: + # https://github.com/supercharge/mongodb-github-action#with-authentication-mongodb---auth-flag + # Apparently, the solution is to write a script that sets up the user beforehand. + #mongodb-replica-set: test-rs - name: Test DB uses: actions-rs/cargo@v1 diff --git a/README.md b/README.md index be44768c2..605285d56 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,16 @@ The changelog can be created using the following command (requires the [`convent ```sh conventional-changelog -p conventionalcommits -i CHANGELOG.md -s ``` + +## Docker deployment configuration of credentials through environment variables + +Docker compose will automatically load credentials for different services from a `.env` file that must either be located in the same directory as the `docker-compose.yml` file, or specified using the `--env-file` flag. You therefore must create such a file before you do a `docker compose up`. An example `.env` file could look like this: + +```ini +MONGODB_USERNAME=root +MONGODB_PASSWORD=root +INFLUXDB_USERNAME=root +INFLUXDB_PASSWORD=password +JWT_PASSWORD=password +JWT_SALT=saltines +``` diff --git a/config.template.toml b/config.template.toml deleted file mode 100644 index 47a93a564..000000000 --- a/config.template.toml +++ /dev/null @@ -1,90 +0,0 @@ -[mongodb] -### Listening address of the MongoDB instance. -conn_str = "mongodb://localhost:27017" - -### MongoDB credentials. These fields are ignored if the connection string already contains credentials. -username = "root" -password = "root" - -### Chronicle allows different database names, so multiple -### networks can run within the same MongoDB instance. -database_name = "chronicle" - -### The minimum amount of connections in the pool. -min_pool_size = 2 - -[influxdb] -### Whether influx time-series data will be written. -metrics_enabled = true -analytics_enabled = true - -### URL pointing to the InfluxDB instance. -url = "http://localhost:8086" - -### The database name used for metrics. -metrics_database_name = "chronicle_metrics" - -### The database name used for analytics. -analytics_database_name = "chronicle_analytics" - -### InfluxDb basic credentials. -username = "root" -password = "password" - -[api] -### Whether API requests will be served. -enabled = true - -### API listening port. -port = 8042 - -### CORS. -allow_origins = "0.0.0.0" - -### JsonWebToken (JWT) credentials. -password_hash = "f36605441dd3b99a0448bc76f51f0e619f47051989cfcbf2fef18670f21799ad" # "password" -password_salt = "saltines" -jwt_expiration = "72h" - -### Public API routes. -public_routes = [ - # Activated APIs. - "api/core/v2/*", -] - -### Maximum number of records returned by a single API call -max_page_size = 1000 - -[api.argon_config] -### The length of the resulting hash. -hash_length = 32 -### The number of lanes in parallel. -parallelism = 1 -### The amount of memory requested (KB). -mem_cost = 4096 -### The number of passes. -iterations = 3 -### The variant. -variant = "argon2i" -### The version. -version = "0x13" - -[inx] -### Whether INX is used for writing data. -enabled = true - -### Listening address of the node's INX interface. -connect_url = "http://localhost:9029" - -### Time to wait until a new connection attempt is made. -connection_retry_interval = "5s" - -### Maximum number of tries to establish an INX connection. -connection_retry_count = 30 - -[loki] -### Whether Grafana Loki is used for writing logs. -enabled = true - -### The Grafana Loki connection URL. -connect_url = "http://localhost:3100" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 621897d03..692ead843 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,8 +8,8 @@ services: volumes: - ./data/chronicle/mongodb:/data/db environment: - - MONGO_INITDB_ROOT_USERNAME=root - - MONGO_INITDB_ROOT_PASSWORD=root + - MONGO_INITDB_ROOT_USERNAME=${MONGODB_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_PASSWORD} ports: - 27017:27017 @@ -29,17 +29,18 @@ services: ports: - "8042:8042/tcp" # REST API - "9100:9100/tcp" # Metrics - environment: - - RUST_LOG=warn,inx_chronicle=debug tty: true command: - - "--config=config.toml" - - "--inx-url=http://hornet:9029" - "--mongodb-conn-str=mongodb://mongo:27017" + - "--mongodb-username=${MONGODB_USERNAME}" + - "--mongodb-password=${MONGODB_PASSWORD}" - "--influxdb-url=http://influx:8086" + - "--influxdb-username=${INFLUXDB_USERNAME}" + - "--influxdb-password=${INFLUXDB_PASSWORD}" + - "--inx-url=http://hornet:9029" + - "--jwt-password=${JWT_PASSWORD}" + - "--jwt-salt=${JWT_SALT}" - "--loki-url=http://loki:3100" - volumes: - - ../config.template.toml:/app/config.toml:ro influx: image: influxdb:1.8 @@ -48,8 +49,8 @@ services: - ./data/chronicle/influxdb:/var/lib/influxdb - ./assets/influxdb/init.iql:/docker-entrypoint-initdb.d/influx_init.iql environment: - - INFLUXDB_ADMIN_USER=root - - INFLUXDB_ADMIN_PASSWORD=password + - INFLUXDB_ADMIN_USER=${INFLUXDB_USERNAME} + - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_PASSWORD} - INFLUXDB_HTTP_AUTH_ENABLED=true # The following variables are used to scale InfluxDB to larger amounts of data. It remains to be seen how this # will affect performance in the long run. diff --git a/src/bin/inx-chronicle/api/auth.rs b/src/bin/inx-chronicle/api/auth.rs index 336126b85..9df4c6cd3 100644 --- a/src/bin/inx-chronicle/api/auth.rs +++ b/src/bin/inx-chronicle/api/auth.rs @@ -9,14 +9,14 @@ use axum::{ TypedHeader, }; -use super::{config::ApiData, error::RequestError, ApiError, AuthError}; +use super::{config::ApiConfigData, error::RequestError, ApiError, AuthError}; pub struct Auth; #[async_trait] impl FromRequestParts for Auth where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -24,7 +24,7 @@ where // Unwrap: ::Rejection = Infallable let OriginalUri(uri) = OriginalUri::from_request_parts(req, state).await.unwrap(); - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); if config.public_routes.is_match(&uri.to_string()) { return Ok(Auth); @@ -37,10 +37,10 @@ where jwt.validate( Validation::default() - .with_issuer(ApiData::ISSUER) - .with_audience(ApiData::AUDIENCE) + .with_issuer(ApiConfigData::ISSUER) + .with_audience(ApiConfigData::AUDIENCE) .validate_nbf(true), - config.secret_key.as_ref(), + config.jwt_secret_key.as_ref(), ) .map_err(AuthError::InvalidJwt)?; diff --git a/src/bin/inx-chronicle/api/config.rs b/src/bin/inx-chronicle/api/config.rs index 527f605e8..3b3a451f4 100644 --- a/src/bin/inx-chronicle/api/config.rs +++ b/src/bin/inx-chronicle/api/config.rs @@ -10,6 +10,15 @@ use tower_http::cors::AllowOrigin; use super::{error::ConfigError, SecretKey}; +pub const DEFAULT_ENABLED: bool = true; +pub const DEFAULT_PORT: u16 = 8042; +pub const DEFAULT_ALLOW_ORIGINS: &str = "0.0.0.0"; +pub const DEFAULT_PUBLIC_ROUTES: &str = "api/core/v2/*"; +pub const DEFAULT_MAX_PAGE_SIZE: usize = 1000; +pub const DEFAULT_JWT_PASSWORD: &str = "password"; +pub const DEFAULT_JWT_SALT: &str = "saltines"; +pub const DEFAULT_JWT_EXPIRATION: &str = "72h"; + /// API configuration #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] @@ -17,64 +26,67 @@ pub struct ApiConfig { pub enabled: bool, pub port: u16, pub allow_origins: SingleOrMultiple, - pub password_hash: String, - pub password_salt: String, - #[serde(with = "humantime_serde")] - pub jwt_expiration: Duration, pub public_routes: Vec, - pub identity_path: Option, pub max_page_size: usize, - pub argon_config: ArgonConfig, + pub jwt_password: String, + pub jwt_salt: String, + pub jwt_identity_file: Option, + #[serde(with = "humantime_serde")] + pub jwt_expiration: Duration, } impl Default for ApiConfig { fn default() -> Self { Self { - enabled: true, - port: 8042, - allow_origins: "*".to_string().into(), - password_hash: "c42cf2be3a442a29d8cd827a27099b0c".to_string(), - password_salt: "saltines".to_string(), - // 72 hours - jwt_expiration: Duration::from_secs(72 * 60 * 60), - public_routes: Default::default(), - identity_path: None, - max_page_size: 1000, - argon_config: Default::default(), + enabled: DEFAULT_ENABLED, + port: DEFAULT_PORT, + allow_origins: SingleOrMultiple::Single(DEFAULT_ALLOW_ORIGINS.to_string()), + public_routes: vec![DEFAULT_PUBLIC_ROUTES.to_string()], + max_page_size: DEFAULT_MAX_PAGE_SIZE, + jwt_identity_file: None, + jwt_password: DEFAULT_JWT_PASSWORD.to_string(), + jwt_salt: DEFAULT_JWT_SALT.to_string(), + jwt_expiration: DEFAULT_JWT_EXPIRATION.parse::().unwrap().into(), } } } #[derive(Clone, Debug)] -pub struct ApiData { +pub struct ApiConfigData { pub port: u16, pub allow_origins: AllowOrigin, - pub password_hash: Vec, - pub password_salt: String, - pub jwt_expiration: Duration, pub public_routes: RegexSet, - pub secret_key: SecretKey, pub max_page_size: usize, - pub argon_config: ArgonConfig, + pub jwt_password_hash: Vec, + pub jwt_password_salt: String, + pub jwt_secret_key: SecretKey, + pub jwt_expiration: Duration, + pub jwt_argon_config: JwtArgonConfig, } -impl ApiData { +impl ApiConfigData { pub const ISSUER: &'static str = "chronicle"; pub const AUDIENCE: &'static str = "api"; } -impl TryFrom for ApiData { +impl TryFrom for ApiConfigData { type Error = ConfigError; fn try_from(config: ApiConfig) -> Result { Ok(Self { port: config.port, allow_origins: AllowOrigin::try_from(config.allow_origins)?, - password_hash: hex::decode(config.password_hash)?, - password_salt: config.password_salt, - jwt_expiration: config.jwt_expiration, public_routes: RegexSet::new(config.public_routes.iter().map(route_to_regex).collect::>())?, - secret_key: match &config.identity_path { + max_page_size: config.max_page_size, + jwt_password_hash: argon2::hash_raw( + config.jwt_password.as_bytes(), + config.jwt_salt.as_bytes(), + &Into::into(&JwtArgonConfig::default()), + ) + // TODO: Replace this once we switch to a better error lib + .expect("invalid JWT config"), + jwt_password_salt: config.jwt_salt, + jwt_secret_key: match &config.jwt_identity_file { Some(path) => SecretKey::from_file(path)?, None => { if let Ok(path) = std::env::var("IDENTITY_PATH") { @@ -84,8 +96,8 @@ impl TryFrom for ApiData { } } }, - max_page_size: config.max_page_size, - argon_config: config.argon_config, + jwt_expiration: config.jwt_expiration, + jwt_argon_config: JwtArgonConfig::default(), }) } } @@ -130,9 +142,27 @@ impl TryFrom> for AllowOrigin { } } +impl Default for SingleOrMultiple { + fn default() -> Self { + Self::Single(Default::default()) + } +} + +impl From<&Vec> for SingleOrMultiple { + fn from(value: &Vec) -> Self { + if value.is_empty() { + unreachable!("Vec must have single or multiple elements") + } else if value.len() == 1 { + Self::Single(value[0].clone()) + } else { + Self::Multiple(value.to_vec()) + } + } +} + #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default)] -pub struct ArgonConfig { +pub struct JwtArgonConfig { /// The length of the resulting hash. hash_length: u32, /// The number of lanes in parallel. @@ -149,7 +179,7 @@ pub struct ArgonConfig { version: argon2::Version, } -impl Default for ArgonConfig { +impl Default for JwtArgonConfig { fn default() -> Self { Self { hash_length: 32, @@ -162,8 +192,8 @@ impl Default for ArgonConfig { } } -impl<'a> From<&'a ArgonConfig> for argon2::Config<'a> { - fn from(val: &'a ArgonConfig) -> Self { +impl<'a> From<&'a JwtArgonConfig> for argon2::Config<'a> { + fn from(val: &'a JwtArgonConfig) -> Self { Self { ad: &[], hash_length: val.hash_length, diff --git a/src/bin/inx-chronicle/api/extractors.rs b/src/bin/inx-chronicle/api/extractors.rs index 8ca8a49bd..a60a3419b 100644 --- a/src/bin/inx-chronicle/api/extractors.rs +++ b/src/bin/inx-chronicle/api/extractors.rs @@ -6,7 +6,7 @@ use axum::extract::{FromRef, FromRequestParts, Query}; use serde::Deserialize; use super::{ - config::ApiData, + config::ApiConfigData, error::{ApiError, RequestError}, DEFAULT_PAGE_SIZE, }; @@ -30,7 +30,7 @@ impl Default for Pagination { #[async_trait] impl FromRequestParts for Pagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -38,7 +38,7 @@ where let Query(mut pagination) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); pagination.page_size = pagination.page_size.min(config.max_page_size); Ok(pagination) } @@ -117,14 +117,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - Pagination::from_request(req, &config).await.unwrap(), + Pagination::from_request(req, &data).await.unwrap(), Pagination { page_size: 1000, ..Default::default() diff --git a/src/bin/inx-chronicle/api/mod.rs b/src/bin/inx-chronicle/api/mod.rs index 5b1e1bf44..8f282e9f1 100644 --- a/src/bin/inx-chronicle/api/mod.rs +++ b/src/bin/inx-chronicle/api/mod.rs @@ -14,7 +14,7 @@ mod secret_key; #[macro_use] mod responses; mod auth; -mod config; +pub mod config; mod router; mod routes; @@ -30,7 +30,7 @@ use tracing::info; use self::routes::routes; pub use self::{ - config::{ApiConfig, ApiData}, + config::{ApiConfig, ApiConfigData}, error::{ApiError, ApiResult, AuthError, ConfigError}, secret_key::SecretKey, }; @@ -41,7 +41,7 @@ pub const DEFAULT_PAGE_SIZE: usize = 100; #[derive(Debug, Clone)] pub struct ApiWorker { db: MongoDb, - api_data: ApiData, + api_data: ApiConfigData, } impl ApiWorker { diff --git a/src/bin/inx-chronicle/api/router.rs b/src/bin/inx-chronicle/api/router.rs index a3237d3d4..59492bca5 100644 --- a/src/bin/inx-chronicle/api/router.rs +++ b/src/bin/inx-chronicle/api/router.rs @@ -27,7 +27,7 @@ use hyper::{Body, Request}; use regex::RegexSet; use tower::{Layer, Service}; -use super::{ApiData, ApiWorker}; +use super::{ApiConfigData, ApiWorker}; #[derive(Clone, Debug, Default)] pub struct RouteNode { @@ -95,7 +95,7 @@ impl FromRef> for MongoDb { } } -impl FromRef> for ApiData { +impl FromRef> for ApiConfigData { fn from_ref(input: &RouterState) -> Self { input.inner.api_data.clone() } diff --git a/src/bin/inx-chronicle/api/routes.rs b/src/bin/inx-chronicle/api/routes.rs index 69ea6a682..5313245ed 100644 --- a/src/bin/inx-chronicle/api/routes.rs +++ b/src/bin/inx-chronicle/api/routes.rs @@ -21,7 +21,7 @@ use time::{Duration, OffsetDateTime}; use super::{ auth::Auth, - config::ApiData, + config::ApiConfigData, error::{ApiError, MissingError, UnimplementedError}, extractors::ListRoutesQuery, responses::RoutesResponse, @@ -37,7 +37,7 @@ const ALWAYS_AVAILABLE_ROUTES: &[&str] = &["/health", "/login", "/routes"]; // sufficient time to catch up with the node that it is connected too. The current milestone interval is 5 seconds. const STALE_MILESTONE_DURATION: Duration = Duration::minutes(5); -pub fn routes(api_data: ApiData) -> Router { +pub fn routes(api_data: ApiConfigData) -> Router { #[allow(unused_mut)] let mut router = Router::new(); @@ -62,17 +62,24 @@ struct LoginInfo { password: String, } -async fn login(State(config): State, Json(LoginInfo { password }): Json) -> ApiResult { +async fn login( + State(config_data): State, + Json(LoginInfo { password }): Json, +) -> ApiResult { if password_verify( password.as_bytes(), - config.password_salt.as_bytes(), - &config.password_hash, - Into::into(&config.argon_config), + config_data.jwt_password_salt.as_bytes(), + &config_data.jwt_password_hash, + Into::into(&config_data.jwt_argon_config), )? { let jwt = JsonWebToken::new( - Claims::new(ApiData::ISSUER, uuid::Uuid::new_v4().to_string(), ApiData::AUDIENCE)? - .expires_after_duration(config.jwt_expiration)?, - config.secret_key.as_ref(), + Claims::new( + ApiConfigData::ISSUER, + uuid::Uuid::new_v4().to_string(), + ApiConfigData::AUDIENCE, + )? + .expires_after_duration(config_data.jwt_expiration)?, + config_data.jwt_secret_key.as_ref(), )?; Ok(format!("Bearer {}", jwt)) @@ -108,10 +115,10 @@ async fn list_routes( jwt.validate( Validation::default() - .with_issuer(ApiData::ISSUER) - .with_audience(ApiData::AUDIENCE) + .with_issuer(ApiConfigData::ISSUER) + .with_audience(ApiConfigData::AUDIENCE) .validate_nbf(true), - state.inner.api_data.secret_key.as_ref(), + state.inner.api_data.jwt_secret_key.as_ref(), ) .map_err(AuthError::InvalidJwt)?; diff --git a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs index 5c4bae8fe..e67c520b8 100644 --- a/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/explorer/extractors.rs @@ -14,7 +14,7 @@ use chronicle::{ }; use serde::Deserialize; -use crate::api::{config::ApiData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; +use crate::api::{config::ApiConfigData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct LedgerUpdatesByAddressPagination { @@ -73,7 +73,7 @@ impl Display for LedgerUpdatesByAddressCursor { #[async_trait] impl FromRequestParts for LedgerUpdatesByAddressPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -81,7 +81,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -155,7 +155,7 @@ impl Display for LedgerUpdatesByMilestoneCursor { #[async_trait] impl FromRequestParts for LedgerUpdatesByMilestonePagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -163,7 +163,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (page_size, cursor) = if let Some(cursor) = query.cursor { let cursor: LedgerUpdatesByMilestoneCursor = cursor.parse()?; @@ -227,7 +227,7 @@ impl Display for MilestonesCursor { #[async_trait] impl FromRequestParts for MilestonesPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -235,7 +235,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); if matches!((query.start_timestamp, query.end_timestamp), (Some(start), Some(end)) if end < start) { return Err(ApiError::from(RequestError::BadTimeRange)); @@ -285,7 +285,7 @@ impl Default for RichestAddressesQuery { #[async_trait] impl FromRequestParts for RichestAddressesQuery where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -293,7 +293,7 @@ where let Query(mut query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); query.top = query.top.min(config.max_page_size); Ok(query) } @@ -383,7 +383,7 @@ impl Display for BlocksByMilestoneCursor { #[async_trait] impl FromRequestParts for BlocksByMilestoneIndexPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -391,7 +391,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -431,7 +431,7 @@ pub struct BlocksByMilestoneIdPaginationQuery { #[async_trait] impl FromRequestParts for BlocksByMilestoneIdPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -439,7 +439,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let sort = query .sort @@ -494,14 +494,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/ledger/updates/by-address/0x00?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - LedgerUpdatesByAddressPagination::from_request(req, &config) + LedgerUpdatesByAddressPagination::from_request(req, &data) .await .unwrap(), LedgerUpdatesByAddressPagination { @@ -517,7 +521,7 @@ mod test { .body(()) .unwrap(); assert_eq!( - LedgerUpdatesByMilestonePagination::from_request(req, &config) + LedgerUpdatesByMilestonePagination::from_request(req, &data) .await .unwrap(), LedgerUpdatesByMilestonePagination { diff --git a/src/bin/inx-chronicle/api/stardust/explorer/routes.rs b/src/bin/inx-chronicle/api/stardust/explorer/routes.rs index 91155434b..8be379d36 100644 --- a/src/bin/inx-chronicle/api/stardust/explorer/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/explorer/routes.rs @@ -37,14 +37,14 @@ use crate::api::{ error::{CorruptStateError, MissingError, RequestError}, extractors::Pagination, router::{Router, RouterState}, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new() .route("/balance/:address", get(balance)) diff --git a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs index 131d37726..15af1b1dc 100644 --- a/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs +++ b/src/bin/inx-chronicle/api/stardust/indexer/extractors.rs @@ -16,7 +16,7 @@ use mongodb::bson; use primitive_types::U256; use serde::Deserialize; -use crate::api::{config::ApiData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; +use crate::api::{config::ApiConfigData, error::RequestError, ApiError, DEFAULT_PAGE_SIZE}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct IndexedOutputsPagination @@ -94,7 +94,7 @@ pub struct BasicOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -102,7 +102,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -190,7 +190,7 @@ pub struct AliasOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -198,7 +198,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -275,7 +275,7 @@ pub struct FoundryOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -283,7 +283,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -357,7 +357,7 @@ pub struct NftOutputsPaginationQuery { #[async_trait] impl FromRequestParts for IndexedOutputsPagination where - ApiData: FromRef, + ApiConfigData: FromRef, { type Rejection = ApiError; @@ -365,7 +365,7 @@ where let Query(query) = Query::::from_request_parts(req, state) .await .map_err(RequestError::from)?; - let config = ApiData::from_ref(state); + let config = ApiConfigData::from_ref(state); let (cursor, page_size) = if let Some(cursor) = query.cursor { let cursor: IndexedOutputsCursor = cursor.parse()?; @@ -457,14 +457,18 @@ mod test { #[tokio::test] async fn page_size_clamped() { - let config = ApiData::try_from(ApiConfig::default()).unwrap(); + let config = ApiConfig { + max_page_size: 1000, + ..Default::default() + }; + let data = ApiConfigData::try_from(config).unwrap(); let req = Request::builder() .method("GET") .uri("/outputs/basic?pageSize=9999999") .body(()) .unwrap(); assert_eq!( - IndexedOutputsPagination::::from_request(req, &config) + IndexedOutputsPagination::::from_request(req, &data) .await .unwrap(), IndexedOutputsPagination { diff --git a/src/bin/inx-chronicle/api/stardust/indexer/routes.rs b/src/bin/inx-chronicle/api/stardust/indexer/routes.rs index 6898e1bd5..4b5649a1a 100644 --- a/src/bin/inx-chronicle/api/stardust/indexer/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/indexer/routes.rs @@ -24,14 +24,14 @@ use crate::api::{ error::{MissingError, RequestError}, router::{Router, RouterState}, stardust::indexer::extractors::IndexedOutputsCursor, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new().nest( "/outputs", diff --git a/src/bin/inx-chronicle/api/stardust/poi/routes.rs b/src/bin/inx-chronicle/api/stardust/poi/routes.rs index efe8535f6..f9697242b 100644 --- a/src/bin/inx-chronicle/api/stardust/poi/routes.rs +++ b/src/bin/inx-chronicle/api/stardust/poi/routes.rs @@ -23,14 +23,14 @@ use super::{ use crate::api::{ error::{CorruptStateError, MissingError, RequestError}, router::{Router, RouterState}, - ApiData, ApiResult, + ApiConfigData, ApiResult, }; pub fn routes() -> Router where S: Clone + Send + Sync + 'static, MongoDb: FromRef>, - ApiData: FromRef>, + ApiConfigData: FromRef>, { Router::new() .route( diff --git a/src/bin/inx-chronicle/cli.rs b/src/bin/inx-chronicle/cli.rs index f0e5fca3e..b398c6d32 100644 --- a/src/bin/inx-chronicle/cli.rs +++ b/src/bin/inx-chronicle/cli.rs @@ -1,195 +1,265 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +use chronicle::db::mongodb::config as mongodb; use clap::{Args, Parser, Subcommand, ValueEnum}; -use crate::config::{ChronicleConfig, ConfigError}; +use crate::config::ChronicleConfig; /// Chronicle permanode storage as an INX plugin #[derive(Parser, Debug)] -#[command(author, version, about, next_display_order = None)] +// #[command(author, version, about, next_display_order = None)] +#[command(author, version, about)] pub struct ClArgs { - /// The location of the configuration file. - #[arg(short, long, env = "CONFIG_PATH")] - pub config: Option, - /// Rest API arguments. - #[cfg(feature = "api")] - #[command(flatten)] - pub api: ApiArgs, + /// MongoDb arguments. + #[command(flatten, next_help_heading = "MongoDb")] + pub mongodb: MongoDbArgs, /// InfluxDb arguments. #[cfg(any(feature = "analytics", feature = "metrics"))] - #[command(flatten)] + #[command(flatten, next_help_heading = "InfluxDb")] pub influxdb: InfluxDbArgs, /// INX arguments. #[cfg(feature = "inx")] - #[command(flatten)] + #[command(flatten, next_help_heading = "INX")] pub inx: InxArgs, - /// MongoDb arguments. - #[command(flatten)] - pub mongodb: MongoDbArgs, + /// Rest API arguments. + #[cfg(feature = "api")] + #[command(flatten, next_help_heading = "API")] + pub api: ApiArgs, /// Loki arguments. #[cfg(feature = "loki")] - #[command(flatten)] + #[command(flatten, next_help_heading = "Loki")] pub loki: LokiArgs, /// Subcommands. #[command(subcommand)] pub subcommand: Option, } -#[cfg(feature = "api")] -#[derive(Args, Debug)] -pub struct ApiArgs { - /// Toggle REST API. - #[arg(long, env = "REST_API_ENABLED")] - pub api_enabled: Option, - /// JWT arguments. - #[command(flatten)] - pub jwt: JwtArgs, -} - #[derive(Args, Debug)] -pub struct JwtArgs { - /// The location of the identity file for JWT auth. - #[arg(long = "api-jwt-identity", env = "JWT_IDENTITY_PATH")] - pub identity_path: Option, - /// The password used for JWT authentication. - #[arg(long = "api-jwt-password")] - pub password: Option, +pub struct MongoDbArgs { + /// The MongoDb connection string. + #[arg( + long, + value_name = "CONN_STR", + env = "MONGODB_CONN_STR", + default_value = mongodb::DEFAULT_CONN_STR, + )] + pub mongodb_conn_str: String, + /// The MongoDb username. + #[arg(long, value_name = "USERNAME", env = "MONGODB_USERNAME", default_value = mongodb::DEFAULT_USERNAME)] + pub mongodb_username: String, + /// The MongoDb password. + #[arg(long, value_name = "PASSWORD", env = "MONGODB_PASSWORD", default_value = mongodb::DEFAULT_PASSWORD)] + pub mongodb_password: String, + /// The MongoDb database name. + #[arg(long, value_name = "NAME", default_value = mongodb::DEFAULT_DATABASE_NAME)] + pub mongodb_database_name: String, + /// The MongoDb minimum pool size. + #[arg(long, value_name = "SIZE", default_value_t = mongodb::DEFAULT_MIN_POOL_SIZE)] + pub mongodb_min_pool_size: u32, } -#[cfg(feature = "inx")] -#[derive(Args, Debug)] -pub struct InxArgs { - /// Toggle INX write workflow. - #[arg(long, env = "INX_ENABLED")] - pub inx_enabled: Option, - /// The address of the INX interface provided by the node. - #[arg(long, env = "INX_URL")] - pub inx_url: Option, - /// Milestone at which synchronization should begin. A value of `1` means syncing back until genesis (default). - #[arg(long = "inx-sync-start")] - pub sync_start: Option, +impl From<&MongoDbArgs> for chronicle::db::MongoDbConfig { + fn from(value: &MongoDbArgs) -> Self { + Self { + conn_str: value.mongodb_conn_str.clone(), + username: value.mongodb_username.clone(), + password: value.mongodb_password.clone(), + database_name: value.mongodb_database_name.clone(), + min_pool_size: value.mongodb_min_pool_size, + } + } } -#[derive(Args, Debug)] -pub struct MongoDbArgs { - /// The MongoDB connection string. - #[arg(long, env = "MONGODB_CONN_STR")] - pub mongodb_conn_str: Option, - /// The MongoDB database. - #[arg(long, env = "MONGODB_DATABASE")] - pub mongodb_database: Option, -} +#[cfg(any(feature = "analytics", feature = "metrics"))] +use chronicle::db::influxdb::config as influxdb; #[cfg(any(feature = "analytics", feature = "metrics"))] #[derive(Args, Debug)] pub struct InfluxDbArgs { - /// Toggle InfluxDb time-series metrics writes. - #[arg(long, env = "METRICS_ENABLED")] - pub metrics_enabled: Option, - /// Toggle InfluxDb time-series analytics writes. - #[arg(long, env = "ANALYTICS_ENABLED")] - pub analytics_enabled: Option, /// The url pointing to an InfluxDb instance. - #[arg(long, env = "INFLUXDB_URL")] - pub influxdb_url: Option, + #[arg(long, value_name = "URL", default_value = influxdb::DEFAULT_URL)] + pub influxdb_url: String, + /// The InfluxDb username. + #[arg(long, value_name = "USERNAME", env = "INFLUXDB_USERNAME", default_value = influxdb::DEFAULT_USERNAME)] + pub influxdb_username: String, + /// The InfluxDb password. + #[arg(long, value_name = "PASSWORD", env = "INFLUXDB_PASSWORD", default_value = influxdb::DEFAULT_PASSWORD)] + pub influxdb_password: String, + /// The Analytics database name. + #[cfg(feature = "analytics")] + #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_ANALYTICS_DATABASE_NAME)] + pub analytics_database_name: String, + /// The Metrics database name. + #[cfg(feature = "metrics")] + #[arg(long, value_name = "NAME", default_value = influxdb::DEFAULT_METRICS_DATABASE_NAME)] + pub metrics_database_name: String, + /// Disable InfluxDb time-series analytics writes. + #[cfg(feature = "analytics")] + #[arg(long, default_value_t = !influxdb::DEFAULT_ANALYTICS_ENABLED)] + pub disable_analytics: bool, + /// Disable InfluxDb time-series metrics writes. + #[cfg(feature = "metrics")] + #[arg(long, default_value_t = !influxdb::DEFAULT_METRICS_ENABLED)] + pub disable_metrics: bool, } -#[cfg(feature = "loki")] -#[derive(Args, Debug)] -pub struct LokiArgs { - /// Toggle Grafana Loki log writes. - #[arg(long, env = "LOKI_ENABLED")] - pub loki_enabled: Option, - /// The url pointing to a Grafana Loki instance. - #[arg(long, env = "LOKI_URL")] - pub loki_url: Option, +#[cfg(any(feature = "analytics", feature = "metrics"))] +impl From<&InfluxDbArgs> for chronicle::db::influxdb::InfluxDbConfig { + fn from(value: &InfluxDbArgs) -> Self { + Self { + url: value.influxdb_url.clone(), + username: value.influxdb_username.clone(), + password: value.influxdb_password.clone(), + #[cfg(feature = "analytics")] + analytics_enabled: !value.disable_analytics, + #[cfg(feature = "analytics")] + analytics_database_name: value.analytics_database_name.clone(), + #[cfg(feature = "metrics")] + metrics_enabled: !value.disable_metrics, + #[cfg(feature = "metrics")] + metrics_database_name: value.metrics_database_name.clone(), + } + } } -impl ClArgs { - /// Get a config file with CLI args applied. - pub fn get_config(&self) -> Result { - let mut config = self - .config - .as_ref() - .map(ChronicleConfig::from_file) - .transpose()? - .unwrap_or_default(); +#[cfg(all(feature = "stardust", feature = "inx"))] +use crate::stardust_inx::config as inx; - if let Some(conn_str) = &self.mongodb.mongodb_conn_str { - config.mongodb.conn_str = conn_str.clone(); - } +#[cfg(all(feature = "stardust", feature = "inx"))] +#[derive(Args, Debug)] +pub struct InxArgs { + /// The address of the node INX interface Chronicle tries to connect to - if enabled. + #[arg(long, value_name = "URL", default_value = inx::DEFAULT_URL)] + pub inx_url: String, + /// Time to wait until a new connection attempt is made. + #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = inx::DEFAULT_RETRY_INTERVAL)] + pub inx_retry_interval: std::time::Duration, + /// Maximum number of tries to establish an INX connection. + #[arg(long, value_name = "COUNT", default_value_t = inx::DEFAULT_RETRY_COUNT)] + pub inx_retry_count: usize, + /// Milestone at which synchronization should begin. If set to `1` Chronicle will try to sync back until the + /// genesis block. If set to `0` Chronicle will start syncing from the most recent milestone it received. + #[arg(long, value_name = "START", default_value_t = inx::DEFAULT_SYNC_START)] + pub inx_sync_start: u32, + /// Disable the INX synchronization workflow. + #[arg(long, default_value_t = !inx::DEFAULT_ENABLED)] + pub disable_inx: bool, +} - if let Some(db_name) = &self.mongodb.mongodb_database { - config.mongodb.database_name = db_name.clone(); +#[cfg(all(feature = "stardust", feature = "inx"))] +impl From<&InxArgs> for inx::InxConfig { + fn from(value: &InxArgs) -> Self { + Self { + enabled: !value.disable_inx, + url: value.inx_url.clone(), + conn_retry_interval: value.inx_retry_interval, + conn_retry_count: value.inx_retry_count, + sync_start_milestone: value.inx_sync_start.into(), } + } +} - #[cfg(all(feature = "stardust", feature = "inx"))] - { - if let Some(connect_url) = &self.inx.inx_url { - config.inx.connect_url = connect_url.clone(); - } - if let Some(enabled) = self.inx.inx_enabled { - config.inx.enabled = enabled; - } - if let Some(sync_start) = self.inx.sync_start { - config.inx.sync_start_milestone = sync_start.into(); - } - } +#[cfg(feature = "api")] +use crate::api::config as api; - #[cfg(feature = "analytics")] - { - if let Some(enabled) = self.influxdb.analytics_enabled { - config.influxdb.analytics_enabled = enabled; - } - } +#[cfg(feature = "api")] +#[derive(Args, Debug)] +pub struct ApiArgs { + /// API listening port. + #[arg(long, value_name = "PORT", default_value_t = api::DEFAULT_PORT)] + pub api_port: u16, + /// CORS setting. + #[arg(long = "allow-origin", value_name = "IP", default_value = api::DEFAULT_ALLOW_ORIGINS)] + pub allow_origins: Vec, + /// Public API routes. + #[arg(long = "public-route", value_name = "ROUTE", default_value = api::DEFAULT_PUBLIC_ROUTES)] + pub public_routes: Vec, + /// Maximum number of results returned by a single API call. + #[arg(long, value_name = "SIZE", default_value_t = api::DEFAULT_MAX_PAGE_SIZE)] + pub max_page_size: usize, + /// JWT arguments. + #[command(flatten)] + pub jwt: JwtArgs, + /// Disable REST API. + #[arg(long, default_value_t = !api::DEFAULT_ENABLED)] + pub disable_api: bool, +} - #[cfg(feature = "metrics")] - { - if let Some(enabled) = self.influxdb.metrics_enabled { - config.influxdb.metrics_enabled = enabled; - } +#[cfg(feature = "api")] +impl From<&ApiArgs> for api::ApiConfig { + fn from(value: &ApiArgs) -> Self { + Self { + enabled: !value.disable_api, + port: value.api_port, + allow_origins: (&value.allow_origins).into(), + jwt_password: value.jwt.jwt_password.clone(), + jwt_salt: value.jwt.jwt_salt.clone(), + jwt_identity_file: value.jwt.jwt_identity.clone(), + jwt_expiration: value.jwt.jwt_expiration, + max_page_size: value.max_page_size, + public_routes: value.public_routes.clone(), } + } +} - #[cfg(any(feature = "analytics", feature = "metrics"))] - { - if let Some(url) = &self.influxdb.influxdb_url { - config.influxdb.url = url.clone(); - } - } +#[cfg(feature = "api")] +#[derive(Args, Debug)] +pub struct JwtArgs { + /// The location of the identity file for JWT auth. + #[arg(long, value_name = "FILEPATH", env = "JWT_IDENTITY", default_value = None)] + pub jwt_identity: Option, + /// The password used for JWT authentication. + #[arg(long, value_name = "PASSWORD", env = "JWT_PASSWORD", default_value = api::DEFAULT_JWT_PASSWORD)] + pub jwt_password: String, + /// The salt used for JWT authentication. + #[arg(long, value_name = "SALT", env = "JWT_SALT", default_value = api::DEFAULT_JWT_SALT)] + pub jwt_salt: String, + /// The setting for when the (JWT) token expires. + #[arg(long, value_name = "DURATION", value_parser = parse_duration, default_value = api::DEFAULT_JWT_EXPIRATION)] + pub jwt_expiration: std::time::Duration, +} - #[cfg(feature = "api")] - { - if let Some(password) = &self.api.jwt.password { - config.api.password_hash = hex::encode( - argon2::hash_raw( - password.as_bytes(), - config.api.password_salt.as_bytes(), - &Into::into(&config.api.argon_config), - ) - // TODO: Replace this once we switch to a better error lib - .expect("invalid JWT config"), - ); - } - if let Some(path) = &self.api.jwt.identity_path { - config.api.identity_path.replace(path.clone()); - } - if let Some(enabled) = self.api.api_enabled { - config.api.enabled = enabled; - } - } +#[cfg(feature = "loki")] +#[derive(Args, Debug)] +pub struct LokiArgs { + /// The url pointing to a Grafana Loki instance. + #[arg(long, value_name = "URL", default_value = crate::config::loki::DEFAULT_LOKI_URL)] + pub loki_url: String, + /// Disable Grafana Loki log writes. + #[arg(long, default_value_t = !crate::config::loki::DEFAULT_LOKI_ENABLED)] + pub disable_loki: bool, +} - #[cfg(feature = "loki")] - { - if let Some(connect_url) = &self.loki.loki_url { - config.loki.connect_url = connect_url.clone(); - } - if let Some(enabled) = self.loki.loki_enabled { - config.loki.enabled = enabled; - } +#[cfg(feature = "loki")] +impl From<&LokiArgs> for crate::config::loki::LokiConfig { + fn from(value: &LokiArgs) -> Self { + Self { + enabled: !value.disable_loki, + url: value.loki_url.clone(), } + } +} + +#[cfg(any(all(feature = "stardust", feature = "inx"), feature = "api"))] +fn parse_duration(arg: &str) -> Result { + arg.parse::().map(Into::into) +} - Ok(config) +impl ClArgs { + /// Creates a [`ChronicleConfig`] from the given command-line arguments, environment variables, and defaults. + pub fn get_config(&self) -> ChronicleConfig { + ChronicleConfig { + mongodb: (&self.mongodb).into(), + #[cfg(any(feature = "analytics", feature = "metrics"))] + influxdb: (&self.influxdb).into(), + #[cfg(all(feature = "stardust", feature = "inx"))] + inx: (&self.inx).into(), + #[cfg(feature = "api")] + api: (&self.api).into(), + #[cfg(feature = "loki")] + loki: (&self.loki).into(), + } } /// Process subcommands and return whether the app should early exit. @@ -200,18 +270,18 @@ impl ClArgs { match subcommand { #[cfg(feature = "api")] Subcommands::GenerateJWT => { - use crate::api::ApiData; - let api_data = ApiData::try_from(config.api.clone()).expect("invalid API config"); + use crate::api::ApiConfigData; + let api_data = ApiConfigData::try_from(config.api.clone()).expect("invalid API config"); let claims = auth_helper::jwt::Claims::new( - ApiData::ISSUER, + ApiConfigData::ISSUER, uuid::Uuid::new_v4().to_string(), - ApiData::AUDIENCE, + ApiConfigData::AUDIENCE, ) .unwrap() // Panic: Cannot fail. .expires_after_duration(api_data.jwt_expiration) .map_err(crate::api::AuthError::InvalidJwt)?; let exp_ts = time::OffsetDateTime::from_unix_timestamp(claims.exp.unwrap() as _).unwrap(); - let jwt = auth_helper::jwt::JsonWebToken::new(claims, api_data.secret_key.as_ref()) + let jwt = auth_helper::jwt::JsonWebToken::new(claims, api_data.jwt_secret_key.as_ref()) .map_err(crate::api::AuthError::InvalidJwt)?; tracing::info!("Bearer {}", jwt); tracing::info!( @@ -221,7 +291,7 @@ impl ClArgs { ); return Ok(PostCommand::Exit); } - #[cfg(all(feature = "analytics", feature = "stardust"))] + #[cfg(all(feature = "analytics", feature = "stardust", feature = "inx"))] Subcommands::FillAnalytics { start_milestone, end_milestone, diff --git a/src/bin/inx-chronicle/config.rs b/src/bin/inx-chronicle/config.rs index 27bb3ab9f..0be55cea8 100644 --- a/src/bin/inx-chronicle/config.rs +++ b/src/bin/inx-chronicle/config.rs @@ -1,19 +1,8 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::{fs, path::Path}; - use chronicle::db::MongoDbConfig; use serde::{Deserialize, Serialize}; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum ConfigError { - #[error("failed to read config at '{0}': {1}")] - FileRead(String, std::io::Error), - #[error("toml deserialization failed: {0}")] - TomlDeserialization(toml::de::Error), -} /// Configuration of Chronicle. #[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -27,44 +16,29 @@ pub struct ChronicleConfig { #[cfg(all(feature = "stardust", feature = "inx"))] pub inx: super::stardust_inx::InxConfig, #[cfg(feature = "loki")] - pub loki: LokiConfig, -} - -impl ChronicleConfig { - /// Reads the config from the file located at `path`. - pub fn from_file(path: impl AsRef) -> Result { - fs::read_to_string(&path) - .map_err(|e| ConfigError::FileRead(path.as_ref().display().to_string(), e)) - .and_then(|contents| toml::from_str::(&contents).map_err(ConfigError::TomlDeserialization)) - } + pub loki: loki::LokiConfig, } #[cfg(feature = "loki")] -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct LokiConfig { - pub enabled: bool, - pub connect_url: String, -} - -#[cfg(feature = "loki")] -impl Default for LokiConfig { - fn default() -> Self { - Self { - enabled: true, - connect_url: "http://localhost:3100".to_owned(), - } - } -} - -#[cfg(test)] -mod test { +pub mod loki { use super::*; - #[test] - fn config_file_conformity() -> Result<(), ConfigError> { - let _ = ChronicleConfig::from_file(concat!(env!("CARGO_MANIFEST_DIR"), "/config.template.toml"))?; + pub const DEFAULT_LOKI_ENABLED: bool = true; + pub const DEFAULT_LOKI_URL: &str = "http://localhost:3100"; + + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] + #[serde(default)] + pub struct LokiConfig { + pub enabled: bool, + pub url: String, + } - Ok(()) + impl Default for LokiConfig { + fn default() -> Self { + Self { + enabled: DEFAULT_LOKI_ENABLED, + url: DEFAULT_LOKI_URL.to_string(), + } + } } } diff --git a/src/bin/inx-chronicle/main.rs b/src/bin/inx-chronicle/main.rs index 22f765142..1a3a5a5fb 100644 --- a/src/bin/inx-chronicle/main.rs +++ b/src/bin/inx-chronicle/main.rs @@ -26,12 +26,8 @@ use self::cli::{ClArgs, PostCommand}; async fn main() -> eyre::Result<()> { dotenvy::dotenv().ok(); - std::panic::set_hook(Box::new(|p| { - error!("{}", p); - })); - let cl_args = ClArgs::parse(); - let config = cl_args.get_config()?; + let config = cl_args.get_config(); set_up_logging(&config)?; @@ -58,14 +54,28 @@ async fn main() -> eyre::Result<()> { #[cfg(all(feature = "inx", feature = "stardust"))] if config.inx.enabled { #[cfg(any(feature = "analytics", feature = "metrics"))] - let influx_db = if config.influxdb.analytics_enabled || config.influxdb.metrics_enabled { - info!("Connecting to influx database at address `{}`", config.influxdb.url); + #[allow(unused_mut)] + let mut influx_required = false; + #[cfg(feature = "analytics")] + { + influx_required |= config.influxdb.analytics_enabled; + } + #[cfg(feature = "metrics")] + { + influx_required |= config.influxdb.metrics_enabled; + } + + #[cfg(any(feature = "analytics", feature = "metrics"))] + let influx_db = if influx_required { + info!("Connecting to influx at `{}`", config.influxdb.url); let influx_db = chronicle::db::influxdb::InfluxDb::connect(&config.influxdb).await?; + #[cfg(feature = "analytics")] info!( - "Connected to influx databases `{}` and `{}`", - influx_db.analytics().database_name(), - influx_db.metrics().database_name() + "Connected to influx database `{}`", + influx_db.analytics().database_name() ); + #[cfg(feature = "metrics")] + info!("Connected to influx database `{}`", influx_db.metrics().database_name()); Some(influx_db) } else { None @@ -130,6 +140,10 @@ async fn main() -> eyre::Result<()> { } fn set_up_logging(#[allow(unused)] config: &ChronicleConfig) -> eyre::Result<()> { + std::panic::set_hook(Box::new(|p| { + error!("{}", p); + })); + let registry = tracing_subscriber::registry(); let registry = { @@ -139,7 +153,7 @@ fn set_up_logging(#[allow(unused)] config: &ChronicleConfig) -> eyre::Result<()> }; #[cfg(feature = "loki")] let registry = { - let (layer, task) = tracing_loki::layer(config.loki.connect_url.parse()?, [].into(), [].into())?; + let (layer, task) = tracing_loki::layer(config.loki.url.parse()?, [].into(), [].into())?; tokio::spawn(task); registry.with(layer) }; diff --git a/src/bin/inx-chronicle/stardust_inx/config.rs b/src/bin/inx-chronicle/stardust_inx/config.rs index 888389ec1..342fe0ad5 100644 --- a/src/bin/inx-chronicle/stardust_inx/config.rs +++ b/src/bin/inx-chronicle/stardust_inx/config.rs @@ -6,18 +6,24 @@ use std::time::Duration; use chronicle::types::tangle::MilestoneIndex; use serde::{Deserialize, Serialize}; +pub const DEFAULT_ENABLED: bool = true; +pub const DEFAULT_URL: &str = "http://localhost:9029"; +pub const DEFAULT_RETRY_INTERVAL: &str = "5s"; +pub const DEFAULT_RETRY_COUNT: usize = 30; +pub const DEFAULT_SYNC_START: u32 = 0; + /// Configuration for an INX connection. #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct InxConfig { pub enabled: bool, /// The bind address of node's INX interface. - pub connect_url: String, + pub url: String, /// The time that has to pass until a new connection attempt is made. #[serde(with = "humantime_serde")] - pub connection_retry_interval: Duration, + pub conn_retry_interval: Duration, /// The number of retries when connecting fails. - pub connection_retry_count: usize, + pub conn_retry_count: usize, /// The milestone at which synchronization should begin. pub sync_start_milestone: MilestoneIndex, } @@ -25,11 +31,11 @@ pub struct InxConfig { impl Default for InxConfig { fn default() -> Self { Self { - enabled: true, - connect_url: "http://localhost:9029".into(), - connection_retry_interval: Duration::from_secs(5), - connection_retry_count: 5, - sync_start_milestone: 1.into(), + enabled: DEFAULT_ENABLED, + url: DEFAULT_URL.to_string(), + conn_retry_interval: DEFAULT_RETRY_INTERVAL.parse::().unwrap().into(), + conn_retry_count: DEFAULT_RETRY_COUNT, + sync_start_milestone: DEFAULT_SYNC_START.into(), } } } diff --git a/src/bin/inx-chronicle/stardust_inx/mod.rs b/src/bin/inx-chronicle/stardust_inx/mod.rs index c0ef3d2c1..f0643936c 100644 --- a/src/bin/inx-chronicle/stardust_inx/mod.rs +++ b/src/bin/inx-chronicle/stardust_inx/mod.rs @@ -1,7 +1,7 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -mod config; +pub mod config; mod error; use std::time::Duration; @@ -88,22 +88,22 @@ impl InxWorker { } async fn connect(&self) -> Result { - let url = url::Url::parse(&self.config.connect_url)?; + let url = url::Url::parse(&self.config.url)?; if url.scheme() != "http" { - bail!(InxWorkerError::InvalidAddress(self.config.connect_url.clone())); + bail!(InxWorkerError::InvalidAddress(self.config.url.clone())); } - for i in 0..self.config.connection_retry_count { - match Inx::connect(self.config.connect_url.clone()).await { + for i in 0..self.config.conn_retry_count { + match Inx::connect(self.config.url.clone()).await { Ok(inx_client) => return Ok(inx_client), Err(_) => { warn!( "INX connection failed. Retrying in {}s. {} retries remaining.", - self.config.connection_retry_interval.as_secs(), - self.config.connection_retry_count - i + self.config.conn_retry_interval.as_secs(), + self.config.conn_retry_count - i ); - tokio::time::sleep(self.config.connection_retry_interval).await; + tokio::time::sleep(self.config.conn_retry_interval).await; } } } @@ -135,7 +135,7 @@ impl InxWorker { #[instrument(skip_all, err, level = "trace")] async fn init(&mut self) -> Result<(MilestoneIndex, Inx)> { - info!("Connecting to INX at bind address `{}`.", &self.config.connect_url); + info!("Connecting to INX at bind address `{}`.", &self.config.url); let mut inx = self.connect().await?; info!("Connected to INX."); diff --git a/src/db/influxdb/config.rs b/src/db/influxdb/config.rs new file mode 100644 index 000000000..2523f347d --- /dev/null +++ b/src/db/influxdb/config.rs @@ -0,0 +1,68 @@ +// Copyright 2022 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Holds the `InfluxDb` config and its defaults. + +use serde::{Deserialize, Serialize}; + +/// The default InfluxDb URL to connect to. +pub const DEFAULT_URL: &str = "http://localhost:8086"; +/// The default InfluxDb username. +pub const DEFAULT_USERNAME: &str = "root"; +/// The default InfluxDb password. +pub const DEFAULT_PASSWORD: &str = "password"; +/// The default whether to enable influx analytics writes. +#[cfg(feature = "analytics")] +pub const DEFAULT_ANALYTICS_ENABLED: bool = true; +/// The default name of the analytics database to connect to. +#[cfg(feature = "analytics")] +pub const DEFAULT_ANALYTICS_DATABASE_NAME: &str = "chronicle_analytics"; +/// The default whether to enable influx metrics writes. +#[cfg(feature = "metrics")] +pub const DEFAULT_METRICS_ENABLED: bool = true; +/// The default name of the metrics database to connect to. +#[cfg(feature = "metrics")] +pub const DEFAULT_METRICS_DATABASE_NAME: &str = "chronicle_metrics"; + +/// The influxdb [`influxdb::Client`] config. +#[must_use] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct InfluxDbConfig { + /// The address of the InfluxDb instance. + pub url: String, + /// The InfluxDb username. + pub username: String, + /// The InfluxDb password. + pub password: String, + /// Whether to enable influx analytics writes. + #[cfg(feature = "analytics")] + pub analytics_enabled: bool, + /// The name of the database to insert analytics. + #[cfg(feature = "analytics")] + pub analytics_database_name: String, + /// Whether to enable influx metrics writes. + #[cfg(feature = "metrics")] + pub metrics_enabled: bool, + /// The name of the database to insert metrics. + #[cfg(feature = "metrics")] + pub metrics_database_name: String, +} + +impl Default for InfluxDbConfig { + fn default() -> Self { + Self { + url: DEFAULT_URL.to_string(), + username: DEFAULT_USERNAME.to_string(), + password: DEFAULT_PASSWORD.to_string(), + #[cfg(feature = "analytics")] + analytics_enabled: DEFAULT_ANALYTICS_ENABLED, + #[cfg(feature = "analytics")] + analytics_database_name: DEFAULT_ANALYTICS_DATABASE_NAME.to_string(), + #[cfg(feature = "metrics")] + metrics_enabled: DEFAULT_METRICS_ENABLED, + #[cfg(feature = "metrics")] + metrics_database_name: DEFAULT_METRICS_DATABASE_NAME.to_string(), + } + } +} diff --git a/src/db/influxdb/mod.rs b/src/db/influxdb/mod.rs index 0e5f6d21c..118f7f33b 100644 --- a/src/db/influxdb/mod.rs +++ b/src/db/influxdb/mod.rs @@ -1,14 +1,15 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +pub mod config; mod measurement; use std::ops::Deref; use influxdb::{Client, ReadQuery}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::de::DeserializeOwned; -pub use self::measurement::InfluxDbMeasurement; +pub use self::{config::InfluxDbConfig, measurement::InfluxDbMeasurement}; /// A wrapper for an InfluxDb [`Client`]. #[derive(Clone, Debug)] @@ -48,28 +49,28 @@ impl Deref for InfluxClient { /// A wrapper for the influxdb [`Client`]. #[derive(Clone, Debug)] pub struct InfluxDb { - #[cfg(feature = "metrics")] - metrics_client: InfluxClient, #[cfg(feature = "analytics")] analytics_client: InfluxClient, + #[cfg(feature = "metrics")] + metrics_client: InfluxClient, config: InfluxDbConfig, } impl InfluxDb { /// Create a new influx connection from config. pub async fn connect(config: &InfluxDbConfig) -> Result { - #[cfg(feature = "metrics")] - let metrics_client = { + #[cfg(feature = "analytics")] + let analytics_client = { let client = InfluxClient( - Client::new(&config.url, &config.metrics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.url, &config.analytics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client }; - #[cfg(feature = "analytics")] - let analytics_client = { + #[cfg(feature = "metrics")] + let metrics_client = { let client = InfluxClient( - Client::new(&config.url, &config.analytics_database_name).with_auth(&config.username, &config.password), + Client::new(&config.url, &config.metrics_database_name).with_auth(&config.username, &config.password), ); client.ping().await?; client @@ -83,55 +84,20 @@ impl InfluxDb { }) } - /// Get the metrics client. - #[cfg(feature = "metrics")] - pub fn metrics(&self) -> &InfluxClient { - &self.metrics_client - } - /// Get the analytics client. #[cfg(feature = "analytics")] pub fn analytics(&self) -> &InfluxClient { &self.analytics_client } + /// Get the metrics client. + #[cfg(feature = "metrics")] + pub fn metrics(&self) -> &InfluxClient { + &self.metrics_client + } + /// Get the config used to create the connection. pub fn config(&self) -> &InfluxDbConfig { &self.config } } - -/// The influxdb [`Client`] config. -#[must_use] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct InfluxDbConfig { - /// The address of the InfluxDb instance. - pub url: String, - /// The InfluxDb username. - pub username: String, - /// The InfluxDb password. - pub password: String, - /// The name of the database to insert metrics. - pub metrics_database_name: String, - /// The name of the database to insert analytics. - pub analytics_database_name: String, - /// Whether to enable influx metrics writes. - pub metrics_enabled: bool, - /// Whether to enable influx analytics writes. - pub analytics_enabled: bool, -} - -impl Default for InfluxDbConfig { - fn default() -> Self { - Self { - url: "http://localhost:8086".to_string(), - metrics_database_name: "chronicle_metrics".to_string(), - analytics_database_name: "chronicle_analytics".to_string(), - username: "root".to_string(), - password: "password".to_string(), - metrics_enabled: true, - analytics_enabled: true, - } - } -} diff --git a/src/db/mod.rs b/src/db/mod.rs index ebf150670..530a73161 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -10,6 +10,7 @@ pub mod collections; /// Module containing InfluxDb types and traits. #[cfg(any(feature = "analytics", feature = "metrics"))] pub mod influxdb; -mod mongodb; +/// Module containing MongoDb types and traits. +pub mod mongodb; -pub use self::mongodb::{MongoDb, MongoDbCollection, MongoDbCollectionExt, MongoDbConfig}; +pub use self::mongodb::{config::MongoDbConfig, MongoDb, MongoDbCollection, MongoDbCollectionExt}; diff --git a/src/db/mongodb/collection.rs b/src/db/mongodb/collection.rs index 43b4fd61c..73e09a3f4 100644 --- a/src/db/mongodb/collection.rs +++ b/src/db/mongodb/collection.rs @@ -145,12 +145,13 @@ pub trait MongoDbCollectionExt: MongoDbCollection { } impl MongoDbCollectionExt for T {} -pub(crate) struct InsertResult { - pub(crate) _ignored: usize, +pub struct InsertResult { + _ignored: usize, } +#[allow(missing_docs)] #[async_trait] -pub(crate) trait InsertIgnoreDuplicatesExt { +pub trait InsertIgnoreDuplicatesExt { /// Inserts many records and ignores duplicate key errors. async fn insert_many_ignore_duplicates( &self, diff --git a/src/db/mongodb/config.rs b/src/db/mongodb/config.rs new file mode 100644 index 000000000..7b1f8751f --- /dev/null +++ b/src/db/mongodb/config.rs @@ -0,0 +1,62 @@ +// Copyright 2022 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! Holds the `MongoDb` config and its defaults. + +use mongodb::{ + error::Error, + options::{ConnectionString, HostInfo}, +}; +use serde::{Deserialize, Serialize}; + +/// The default connection string of the database. +pub const DEFAULT_CONN_STR: &str = "mongodb://localhost:27017"; +/// The default MongoDB username. +pub const DEFAULT_USERNAME: &str = "root"; +/// The default MongoDB password. +pub const DEFAULT_PASSWORD: &str = "root"; +/// The default name of the database to connect to. +pub const DEFAULT_DATABASE_NAME: &str = "chronicle"; +/// The default minimum amount of connections in the pool. +pub const DEFAULT_MIN_POOL_SIZE: u32 = 2; + +/// The [`super::MongoDb`] config. +#[must_use] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct MongoDbConfig { + /// The connection string of the database. + pub conn_str: String, + /// The MongoDB username. + pub username: String, + /// The MongoDB password. + pub password: String, + /// The name of the database to connect to. + pub database_name: String, + /// The minimum amount of connections in the pool. + pub min_pool_size: u32, +} + +impl MongoDbConfig { + /// Get the hosts portion of the connection string. + pub fn hosts_str(&self) -> Result { + let hosts = ConnectionString::parse(&self.conn_str)?.host_info; + Ok(match hosts { + HostInfo::HostIdentifiers(hosts) => hosts.iter().map(ToString::to_string).collect::>().join(","), + HostInfo::DnsRecord(hostname) => hostname, + _ => unreachable!(), + }) + } +} + +impl Default for MongoDbConfig { + fn default() -> Self { + Self { + conn_str: DEFAULT_CONN_STR.to_string(), + username: DEFAULT_USERNAME.to_string(), + password: DEFAULT_PASSWORD.to_string(), + database_name: DEFAULT_DATABASE_NAME.to_string(), + min_pool_size: DEFAULT_MIN_POOL_SIZE, + } + } +} diff --git a/src/db/mongodb/mod.rs b/src/db/mongodb/mod.rs index 2848594fc..497b1a766 100644 --- a/src/db/mongodb/mod.rs +++ b/src/db/mongodb/mod.rs @@ -1,22 +1,22 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! Holds the `MongoDb` type and its config. +//! Holds the `MongoDb` type. mod collection; +pub mod config; use std::collections::{HashMap, HashSet}; +use config::MongoDbConfig; use mongodb::{ bson::{doc, Document}, error::Error, - options::{ClientOptions, ConnectionString, Credential, HostInfo}, + options::{ClientOptions, Credential}, Client, }; -use serde::{Deserialize, Serialize}; -pub(crate) use self::collection::InsertIgnoreDuplicatesExt; -pub use self::collection::{MongoDbCollection, MongoDbCollectionExt}; +pub use self::collection::{InsertIgnoreDuplicatesExt, MongoDbCollection, MongoDbCollectionExt}; const DUPLICATE_KEY_CODE: i32 = 11000; @@ -28,24 +28,19 @@ pub struct MongoDb { } impl MongoDb { - const DEFAULT_NAME: &'static str = "chronicle"; - const DEFAULT_CONNECT_STR: &'static str = "mongodb://localhost:27017"; - /// Constructs a [`MongoDb`] by connecting to a MongoDB instance. pub async fn connect(config: &MongoDbConfig) -> Result { let mut client_options = ClientOptions::parse(&config.conn_str).await?; client_options.app_name = Some("Chronicle".to_string()); - client_options.min_pool_size = config.min_pool_size; + client_options.min_pool_size = Some(config.min_pool_size); if client_options.credential.is_none() { - if let (Some(username), Some(password)) = (&config.username, &config.password) { - let credential = Credential::builder() - .username(username.clone()) - .password(password.clone()) - .build(); - client_options.credential = Some(credential); - } + let credential = Credential::builder() + .username(config.username.clone()) + .password(config.password.clone()) + .build(); + client_options.credential = Some(credential); } let client = Client::with_options(client_options)?; @@ -132,44 +127,3 @@ impl MongoDb { self.db.name() } } - -/// The [`MongoDb`] config. -#[must_use] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct MongoDbConfig { - /// The bind address of the database. - pub conn_str: String, - /// The MongoDB username. - pub username: Option, - /// The MongoDB password. - pub password: Option, - /// The name of the database to connect to. - pub database_name: String, - /// The minimum amount of connections in the pool. - pub min_pool_size: Option, -} - -impl MongoDbConfig { - /// Get the hosts portion of the connection string. - pub fn hosts_str(&self) -> Result { - let hosts = ConnectionString::parse(&self.conn_str)?.host_info; - Ok(match hosts { - HostInfo::HostIdentifiers(hosts) => hosts.iter().map(ToString::to_string).collect::>().join(","), - HostInfo::DnsRecord(hostname) => hostname, - _ => unreachable!(), - }) - } -} - -impl Default for MongoDbConfig { - fn default() -> Self { - Self { - conn_str: MongoDb::DEFAULT_CONNECT_STR.to_string(), - username: None, - password: None, - database_name: MongoDb::DEFAULT_NAME.to_string(), - min_pool_size: None, - } - } -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a6bc9992b..5494950b2 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,36 +1,28 @@ // Copyright 2022 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use std::path::Path; - use chronicle::db::{MongoDb, MongoDbCollection, MongoDbConfig}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum TestDbError { - #[error("failed to read config at '{0}': {1}")] - FileRead(String, std::io::Error), - #[error("toml deserialization failed: {0}")] - TomlDeserialization(toml::de::Error), -} #[allow(unused)] pub async fn setup_database(database_name: impl ToString) -> eyre::Result { - let mut config = if let Ok(path) = std::env::var("CONFIG_PATH") { - let val = std::fs::read_to_string(&path) - .map_err(|e| TestDbError::FileRead(AsRef::::as_ref(&path).display().to_string(), e)) - .and_then(|contents| toml::from_str::(&contents).map_err(TestDbError::TomlDeserialization))?; - if let Some(mongodb) = val.get("mongodb").cloned() { - mongodb.try_into().map_err(TestDbError::TomlDeserialization)? - } else { - MongoDbConfig::default() - } - } else { - MongoDbConfig::default() + dotenvy::dotenv().ok(); + + let mut test_config = MongoDbConfig { + database_name: database_name.to_string(), + ..Default::default() + }; + + if let Ok(conn_str) = std::env::var("MONGODB_CONN_STR") { + test_config.conn_str = conn_str; + }; + if let Ok(username) = std::env::var("MONGODB_USERNAME") { + test_config.username = username; + }; + if let Ok(password) = std::env::var("MONGODB_PASSWORD") { + test_config.password = password; }; - config.database_name = database_name.to_string(); - let db = MongoDb::connect(&config).await?; + let db = MongoDb::connect(&test_config).await?; db.clear().await?; Ok(db) }