Skip to content

Commit d3d0037

Browse files
turmelclemturmelclem
authored
Merge pull request #2610 from input-output-hk/ctl/2577-stabilize-cardano-db-v2-aggregator
Stabilization of cardano db v2 (aggregator)
2 parents 028683b + 77385e4 commit d3d0037

File tree

29 files changed

+239
-134
lines changed

29 files changed

+239
-134
lines changed

‎.github/workflows/backward-compatibility.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ on:
1717
description: "Signed entity types parameters (discriminants names in an ordered comma separated list)"
1818
required: true
1919
type: string
20-
default: "CardanoTransactions,CardanoStakeDistribution,CardanoDatabase"
20+
default: "CardanoTransactions,CardanoStakeDistribution,CardanoDatabase,CardanoImmutableFilesFull"
2121
workflow_call:
2222
inputs:
2323
total-releases:
@@ -28,7 +28,7 @@ on:
2828
default: "10.4.1"
2929
signed-entity-types:
3030
type: string
31-
default: "CardanoTransactions,CardanoStakeDistribution,CardanoDatabase"
31+
default: "CardanoTransactions,CardanoStakeDistribution,CardanoDatabase,CardanoImmutableFilesFull"
3232

3333
jobs:
3434
prepare-env-variables:

‎.github/workflows/test-client.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ jobs:
160160
- name: Cardano Database Snapshot / download & restore latest
161161
shell: bash
162162
working-directory: ./bin
163-
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --backend v1 ${{ matrix.extra_args }} 2>&1 | tee cdb-download-output.txt
163+
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --backend v1 --download-dir v1 ${{ matrix.extra_args }} 2>&1 | tee cdb-download-output.txt
164164

165165
- name: Cardano Database Snapshot / verify Cardano node starts successfully
166166
if: runner.os == 'Linux'
@@ -173,19 +173,19 @@ jobs:
173173
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
174174
shell: bash
175175
working-directory: ./bin
176-
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LMDB --commit
176+
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable tools utxo-hd snapshot-converter --db-directory v1/db --cardano-node-version latest --utxo-hd-flavor LMDB --commit
177177

178178
- name: Remove downloaded artifacts to free up disk space (Linux, Windows)
179179
if: runner.os != 'macOS'
180180
shell: bash
181181
working-directory: ./bin
182-
run: rm --force db/immutable/*.{chunk,primary,secondary}
182+
run: rm --force v1/db/immutable/*.{chunk,primary,secondary}
183183

184184
- name: Remove downloaded artifacts to free up disk space (macOs)
185185
if: runner.os == 'macOS'
186186
shell: bash
187187
working-directory: ./bin
188-
run: sudo rm -rf db/
188+
run: sudo rm -rf v1/db/
189189

190190
- name: Mithril Stake Distribution / list and get last hash
191191
shell: bash
@@ -259,13 +259,13 @@ jobs:
259259
if: steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true' || steps.aggregator_capability_windows.outputs.cardano_database_v2_enabled == 'true'
260260
shell: bash
261261
working-directory: ./bin
262-
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download --backend v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-v2-download-output.txt
262+
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download --backend v2 --download-dir v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-v2-download-output.txt
263263

264264
- name: Cardano Database V2 Snapshot / verify immutables
265265
if: steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true' || steps.aggregator_capability_windows.outputs.cardano_database_v2_enabled == 'true'
266266
shell: bash
267267
working-directory: ./bin
268-
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db verify --backend v2 --db-dir db_v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH | tee cdb-v2-verify-output.txt
268+
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db verify --backend v2 --db-dir v2/db $CARDANO_DATABASE_V2_SNAPSHOT_HASH | tee cdb-v2-verify-output.txt
269269

270270
- name: Cardano Database V2 Snapshot / verify Cardano node starts successfully
271271
if: runner.os == 'Linux' && steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true'
@@ -329,16 +329,16 @@ jobs:
329329
330330
- name: Cardano Database Snapshot / download & restore latest
331331
shell: bash
332-
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --backend v1 --download-dir /app/data ${{ matrix.extra_args }}
332+
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --backend v1 --download-dir /app/data/v1 ${{ matrix.extra_args }}
333333

334334
- name: Ledger state snapshot conversion from InMemory to LMDB
335335
if: matrix.extra_args == '--include-ancillary'
336336
shell: bash
337-
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --unstable tools utxo-hd snapshot-converter --db-directory /app/data/db --cardano-node-version latest --utxo-hd-flavor LMDB --commit
337+
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --unstable tools utxo-hd snapshot-converter --db-directory /app/data/v1/db --cardano-node-version latest --utxo-hd-flavor LMDB --commit
338338

339339
- name: Remove downloaded artifacts to free up disk space
340340
shell: bash
341-
run: sudo rm -rf $PWD/data/db/
341+
run: sudo rm -rf $PWD/data/v1/db/
342342

343343
- name: Mithril Stake Distribution / list and get last hash
344344
shell: bash
@@ -401,12 +401,12 @@ jobs:
401401
- name: Cardano Database V2 Snapshot / download & restore latest (Full restoration)
402402
if: steps.aggregator_capability.outputs.cardano_database_v2_enabled == 'true'
403403
shell: bash
404-
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download --backend v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH --download-dir /app/data ${{ matrix.extra_args }}
404+
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download --backend v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH --download-dir /app/data/v2 ${{ matrix.extra_args }}
405405

406406
- name: Cardano Database V2 Snapshot / verify immutables
407407
if: steps.aggregator_capability.outputs.cardano_database_v2_enabled == 'true'
408408
shell: bash
409-
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db verify --backend v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH --db-dir /app/data/db_v2
409+
run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db verify --backend v2 $CARDANO_DATABASE_V2_SNAPSHOT_HASH --db-dir /app/data/v2/db
410410

411411
test-mithril-client-wasm:
412412
strategy:

‎Cargo.lock

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/website/root/manual/develop/nodes/mithril-aggregator.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ Here is a list of the available parameters for the serve command:
527527
| `era_reader_adapter_type` | `--era-reader-adapter-type` | - | `ERA_READER_ADAPTER_TYPE` | Era reader adapter type that can be `cardano-chain`, `file` or `bootstrap`. | `bootstrap` | - | - |
528528
| `era_reader_adapter_params` | `--era-reader-adapter-params` | - | `ERA_READER_ADAPTER_PARAMS` | Era reader adapter params that is an optional JSON encoded parameters structure that is expected depending on the `era_reader_adapter_type` parameter | - | - | - |
529529
| `ancillary_files_signer_config` | - | - | `ANCILLARY_FILES_SIGNER_CONFIG` | Configuration of the ancillary files signer<br/><br/>Can either be a secret key or a key stored in a Google Cloud Platform KMS account.<br/><br/>**IMPORTANT**: The cryptographic scheme used is ED25519 | - | - secret-key:<br/>`{ "type": "secret-key", "secret_key": "136372c3138312c3138382c3130352c3233312c3135" }`<br/>- Gcp kms:<br/>`{ "type": "gcp-kms", "resource_name": "projects/project_name/locations/_location_name/keyRings/key_ring_name/cryptoKeys/key_name/cryptoKeyVersions/key_version" }` | - |
530-
| `signed_entity_types` | `--signed-entity-types` | - | `SIGNED_ENTITY_TYPES` | Signed entity types parameters (discriminants names in an ordered comma separated list) | - | `MithrilStakeDistribution,CardanoImmutableFilesFull,CardanoStakeDistribution` | - |
530+
| `signed_entity_types` | `--signed-entity-types` | - | `SIGNED_ENTITY_TYPES` | Signed entity types parameters (discriminants names in an ordered comma separated list) | - | `MithrilStakeDistribution,CardanoImmutableFilesFull,CardanoStakeDistribution,CardanoDatabase,CardanoTransactions` | - |
531531
| `snapshot_compression_algorithm` | `--snapshot-compression-algorithm` | - | `SNAPSHOT_COMPRESSION_ALGORITHM` | Compression algorithm of the snapshot archive | `zstandard` | `gzip` or `zstandard` | - |
532532
| `zstandard_parameters` | - | - | `ZSTANDARD_PARAMETERS__LEVEL` and `ZSTANDARD_PARAMETERS__NUMBER_OF_WORKERS` | Zstandard specific parameters | - | `{ level: 9, number_of_workers: 4 }` | - |
533533
| `allow_unparsable_block` | `--allow-unparsable-block` | - | `ALLOW_UNPARSABLE_BLOCK` | If set no error is returned in case of unparsable block and an error log is written instead. Will be ignored on (pre)production networks. | `false` | - | - |

‎docs/website/root/manual/develop/nodes/mithril-client.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -578,7 +578,7 @@ Here is a list of the available parameters:
578578

579579
| Parameter | Command line (long) | Command line (short) | Environment variable | Description | Default value | Example | Mandatory |
580580
| -------------------------- | ---------------------------- | :------------------: | -------------------------- | ------------------------------------------------------------------------------- | ------------- | ------- | :----------------: |
581-
| `backend` | `--backend` | `-b` | - | - | `v1` | - | - |
581+
| `backend` | `--backend` | `-b` | - | - | `v2` | - | - |
582582
| `json` | `--json` | - | - | Enable JSON output for command results | `false` | - | - |
583583
| `digest` | - | - | - | Digest of the Cardano db snapshot to verify or `latest` for the latest artifact | - | - | :heavy_check_mark: |
584584
| `db_dir` | `--db-dir` | - | - | Directory from where the immutable will be verified | - | - | - |

‎docs/website/versioned_docs/version-maintained/manual/getting-started/bootstrap-cardano-node.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,9 @@ You will see more information about the snapshot:
446446
447447
### Step 4: Download the selected Cardano DB snapshot
448448
449-
To download the selected snapshot from the remote location to your remote location, run:
449+
#### Step 4.1 : Download the full Cardano node database
450+
451+
To download the full selected snapshot from the remote location to your local location, run:
450452
451453
```bash
452454
./mithril-client cardano-db download --include-ancillary $SNAPSHOT_DIGEST
@@ -493,6 +495,21 @@ Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by runni
493495
mithril-client --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version 10.4.1 --utxo-hd-flavor Legacy --cardano-network preview --commit
494496
```
495497
498+
#### Step 4.2 : Download a partial range of immutable files
499+
500+
To download a range of immutable files (e.g. from 17,000 to 18,000) from the remote location to your local location, prepare the following environment variables:
501+
502+
```bash
503+
export IMMUTABLE_START=17000
504+
export IMMUTABLE_END=18000
505+
```
506+
507+
And run the command:
508+
509+
```bash
510+
./mithril-client cardano-db download $SNAPSHOT_DIGEST --include-ancillary --backend v2 --start $IMMUTABLE_START --end $IMMUTABLE_END
511+
```
512+
496513
### Step 5 (optional): Convert the ledger state snapshot to another flavor
497514
498515
After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor (e.g., `LMDB` or `Legacy`) using the Mithril client `tools utxo-hd snapshot-converter` command.

‎mithril-aggregator/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "mithril-aggregator"
3-
version = "0.7.70"
3+
version = "0.7.71"
44
description = "A Mithril Aggregator server"
55
authors = { workspace = true }
66
edition = { workspace = true }

‎mithril-aggregator/src/configuration.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -495,9 +495,8 @@ pub struct ServeCommandConfiguration {
495495
/// Signed entity types parameters (discriminants names in an ordered, case-sensitive, comma
496496
/// separated list).
497497
///
498-
/// The values `MithrilStakeDistribution` and `CardanoImmutableFilesFull` are prepended
499-
/// automatically to the list.
500-
#[example = "`MithrilStakeDistribution,CardanoImmutableFilesFull,CardanoStakeDistribution`"]
498+
/// The value `MithrilStakeDistribution` is prepended is automatically to the list.
499+
#[example = "`CardanoImmutableFilesFull,CardanoStakeDistribution,CardanoDatabase,CardanoTransactions`"]
501500
pub signed_entity_types: Option<String>,
502501

503502
/// Compression algorithm used for the snapshot archive artifacts.

0 commit comments

Comments
 (0)