feat(analytics): separate base token activity analytic into booked and transferred counts

docker/assets/grafana/dashboards/analytics_dashboard.json

@@ -1065,6 +1065,7 @@
         "type": "influxdb",
         "uid": "PE5723DBC504634E6"
       },
+      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1114,7 +1115,7 @@
               }
             ]
           },
-          "unit": "SMR"
+          "unit": "locale"
         },
         "overrides": []
       },
@@ -1139,7 +1140,136 @@
       },
       "targets": [
         {
-          "alias": "Number of Tokens",
+          "alias": "Number of Booked SMR Tokens",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "PE5723DBC504634E6"
+          },
+          "groupBy": [
+            {
+              "params": [
+                "$aggregation_interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "stardust_base_token_activity",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "booked_value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              },
+              {
+                "params": [
+                  " / 1000000"
+                ],
+                "type": "math"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "title": "SMR Tokens Booked/$aggregation_interval",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "PE5723DBC504634E6"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 100,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "id": 62,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "alias": "Number of Transferred SMR Tokens",
           "datasource": {
             "type": "influxdb",
             "uid": "PE5723DBC504634E6"
@@ -1186,7 +1316,7 @@
           "tags": []
         }
       ],
-      "title": "Tokens Transferred/$aggregation_interval",
+      "title": "SMR Tokens Transferred/$aggregation_interval",
       "type": "timeseries"
     },
     {

src/db/collections/analytics/base_token.rs

@@ -19,6 +19,7 @@ pub struct BaseTokenActivityAnalytics;
 
 #[derive(Copy, Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct BaseTokenActivityAnalyticsResult {
+    pub booked_value: d128,
     pub transferred_value: d128,
 }
 
@@ -53,24 +54,93 @@ impl OutputCollection {
         milestone_index: MilestoneIndex,
     ) -> Result<BaseTokenActivityAnalyticsResult, Error> {
         Ok(self
-            .aggregate(
-                vec![
-                    doc! { "$match": {
-                        "metadata.booked.milestone_index": milestone_index,
-                    } },
-                    doc! { "$group" : {
-                        "_id": null,
-                        "transferred_value": { "$sum": { "$toDecimal": "$output.amount" } },
-                    } },
-                    doc! { "$project": {
-                        "transferred_value": { "$toString": "$transferred_value" },
-                    } },
-                ],
-                None,
-            )
-            .await?
-            .try_next()
-            .await?
-            .unwrap_or_default())
+        .aggregate(
+            vec![
+                // Only consider outputs that were touched in transactions applied by this milestone.
+                doc! { "$match": {
+                    "$or": [
+                        { "metadata.booked.milestone_index": milestone_index },
+                        { "metadata.spent_metadata.spent.milestone_index": milestone_index },
+                    ]
+                } },
+                // Group booked/spent outputs by their booking/spending transaction id and their linked address.
+                // Note that outputs that are booked _and_ spent in this milestone, appear in both groups.
+                doc! { "$facet": {
+                    "booked_outputs": [ 
+                        { "$match": { "metadata.booked.milestone_index": milestone_index } },
+                        { "$group": { 
+                            "_id": {
+                                "tx": "$_id.transaction_id",
+                                "address": "$details.address"
+                            },
+                            "amount": { "$sum": { "$toDecimal": "$output.amount" } },
+                        } }
+                    ],
+                    "spent_outputs": [ 
+                        { "$match": { "metadata.spent_metadata.spent.milestone_index": milestone_index } },
+                        { "$group": { 
+                            "_id": {
+                                "tx": "$metadata.spent_metadata.transaction_id",
+                                "address": "$details.address"
+                            },
+                            "amount": { "$sum": { "$toDecimal": "$output.amount" } },
+                        } }
+                    ],
+                } },
+                // Create a mapping between each booked output group, and all spent output groups.
+                doc! { "$unwind": {
+                    "path": "$booked_outputs",
+                } },
+                // Depending on the current booked output group address and transaction id, determine
+                // if there is a spent output group with the same address. This denotes funds that
+                // are sent back to an input address (and we need to account for that). 
+                doc! { "$project": {
+                    "booked_outputs": 1,
+                    "sent_back_addr": { "$first": {
+                        "$filter": {
+                            "input": "$spent_outputs",
+                            "as": "spent_output",
+                            "cond": { "$and": [ 
+                                { "$eq": ["$$spent_output._id.tx", "$booked_outputs._id.tx"] },
+                                { "$eq": ["$$spent_output._id.address", "$booked_outputs._id.address"] },
+                            ] }
+                        }
+                    } }
+                } },
+                // For the address of the booked output group, get the old (before the transaction) 
+                // and the new (after the transaction) output amount. If that address wasn't an input
+                // address, then assume a virtual input amount of 0.
+                doc! { "$project": {
+                    "new_amount": "$booked_outputs.amount",
+                    "old_amount": { "$ifNull": ["$sent_back_addr.amount", 0] },
+                } },
+                // Sum amounts for various base token analytics.
+                // Notes:
+                // `booked_value`: Sum of all booked output amounts.
+                // `transferred_value`: Sum of all (positive) deltas of amounts per transaction and address.
+                //      - if funds are transferred to a _new_ output address, then the delta is equal to the 
+                //        amount in that new output (due to the virtual input amount of 0);
+                //      - if funds are transferred back to an input address, then the delta is the difference
+                //        between the new amount and the old amount of the corresponding outputs; only if that
+                //        delta is positive (i.e. funds were moved _into_ the linked address) it is counted 
+                //        as a token transfer.   
+                doc! { "$group": {
+                    "_id": null,
+                    "booked_value": { "$sum": "$new_amount" },
+                    "transferred_value": { "$sum": { 
+                        "$cond": [ { "$gt": [ "$new_amount", "$old_amount"] }, { "$subtract": ["$new_amount", "$old_amount"] }, 0   
+                    ] } }
+                } },
+                doc! { "$project": {
+                    "booked_value": { "$toString": "$booked_value" },
+                    "transferred_value": { "$toString": "$transferred_value" },
+                } },
+            ],
+            None,
+        )
+        .await?
+        .try_next()
+        .await?
+        .unwrap_or_default())
     }
 }

src/db/collections/analytics/influx.rs

@@ -23,10 +23,13 @@ impl From<Measurement> for influxdb::WriteQuery {
             Measurement::AddressAnalytics(m) => m
                 .prepare_query("stardust_addresses")
                 .add_field("address_with_balance_count", m.inner.address_with_balance_count),
-            Measurement::BaseTokenActivity(m) => m.prepare_query("stardust_base_token_activity").add_field(
-                "transferred_value",
-                m.inner.transferred_value.to_string().parse::<u64>().unwrap(),
-            ),
+            Measurement::BaseTokenActivity(m) => m
+                .prepare_query("stardust_base_token_activity")
+                .add_field("booked_value", m.inner.booked_value.to_string().parse::<u64>().unwrap())
+                .add_field(
+                    "transferred_value",
+                    m.inner.transferred_value.to_string().parse::<u64>().unwrap(),
+                ),
             Measurement::BlockAnalytics(m) => m
                 .prepare_query("stardust_block_activity")
                 .add_field("transaction_count", m.inner.payload.transaction_count)