diff --git a/.env.example b/.env.example index 6437307..81522f9 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ # REQUIRED SETTINGS # -# If equal to "development", applies configurations on the backend and frontend containers that +# If equal to "development", applies configurations on the backend and frontend containers that # are not suitable for production, related to CORS, secure cookies and https. # Any other value works for a production environment, and will be used by sentry to mark the environment # an error occurred in. @@ -32,10 +32,10 @@ MARBLE_API_URL_CLIENT="http://localhost:8080" # RSA private key, in PEM format, used for for signing authentication tokens. MUST be changed for production. # We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk because multi-line # environment variables are not handled very gracefully. -# +# # If using AUTHENTICATION_JWT_SIGNING_KEY, line breaks should be replaced with '\n': # e.g. AUTHENTICATION_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBA... -# +# # To generate a private key, use `openssl genrsa -out /path/to/private/key.pem 4096`. # If left empty, a key will be regenerated on every app restart, which may cause unexpected logouts. AUTHENTICATION_JWT_SIGNING_KEY= @@ -81,7 +81,7 @@ INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true" CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true" # Configure your Firebase project to allow username and password authentication. -# To retrieve this value, go into the settings of your Firebase project, and register +# To retrieve this value, go into the settings of your Firebase project, and register # a new app from the 'General' tab. FIREBASE_API_KEY= @@ -105,7 +105,7 @@ CREATE_ORG_ADMIN_EMAIL= # Configure the connection details to your Metabase instance. # - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding' -# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as +# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as # a static embedding on Metabase. # - The site url can be on a metabase domain, or can use your self-hosted Metabase instance. METABASE_SITE_URL= @@ -115,7 +115,7 @@ METABASE_GLOBAL_DASHBOARD_ID= # Set up connection details to Convoy to enable webhooks sending. # You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section. # NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors. -CONVOY_API_URL= +CONVOY_API_URL= CONVOY_API_KEY= CONVOY_PROJECT_ID= @@ -124,6 +124,8 @@ CONVOY_PROJECT_ID= # when using docker-compose. YENTE_ELASTICSEARCH_URL=http://marble-es:9200 YENTE_PORT=8000 +YENTE_API_WORKERS=4 +YENTE_QUERY_CONCURRENCY=50 ES_PORT=9200 # Configure access to the Open Sanctions API to use sanction checks @@ -141,6 +143,6 @@ SENTRY_DSN= # If you need to support federated authentication through Firebase, you will need to configure the following settings: # To retrieve those values, go into the settings of your Firebase project, and register a new app from the 'General' tab. -# The two pieces of information below will be provided to you after that. +# The two pieces of information below will be provided to you after that. # FIREBASE_APP_ID= # FIREBASE_AUTH_DOMAIN= diff --git a/docker-compose.yaml b/docker-compose.yaml index 333a9d6..d7dff63 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -160,14 +160,34 @@ services: - marble-es:/usr/share/elasticsearch/data yente: container_name: marble-yente - image: ghcr.io/opensanctions/yente:4.2.1 + image: ghcr.io/opensanctions/yente:4.2.3 depends_on: - elasticsearch + command: + [ + "uvicorn", + "--host=0.0.0.0", + "--workers=${YENTE_API_WORKERS:-4}", + "yente.asgi:app", + ] ports: - ${YENTE_PORT:-8000}:${YENTE_PORT:-8000} environment: YENTE_INDEX_TYPE: elasticsearch - YENTE_INDEX_URL: "${YENTE_ELASTICSEARCH_URL}" + YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200} + YENTE_QUERY_CONCURRENCY: ${YENTE_QUERY_CONCURRENCY:-50} + YENTE_UPDATE_TOKEN: "" + YENTE_AUTO_REINDEX: false + yente-indexer: + container_name: marble-yente-indexer + image: ghcr.io/opensanctions/yente:4.2.3 + depends_on: + - elasticsearch + command: ["yente", "reindex"] + environment: + YENTE_INDEX_TYPE: elasticsearch + YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200} + YENTE_QUERY_CONCURRENCY: "50" YENTE_UPDATE_TOKEN: "" volumes: diff --git a/installation/production_run.md b/installation/production_run.md index 73ffdbc..cb0bfb6 100644 --- a/installation/production_run.md +++ b/installation/production_run.md @@ -131,6 +131,10 @@ Options: > ⚠️ **Note**: While development docker-compose includes Elasticsearch, use a production-grade service for deployment. +It is recommended to deploy the Yente API with multiple workers (see the `docker-compose.yaml` to see how) and disable the indexing process on those with `YENTE_AUTO_REINDEX=false`) so it doesn't impact production workloads and is not duplicated across workers. + +Once automatic background indexing is disabled, you will need to run it (with `yente reindex`) separately through a different container or a scheduled task (cron or systemd timer). + ## Deployment Architecture ### Components