Merge pull request #267 from membermatters/feature/more-prometheus-metrics

Added stats and metrics page to member portal

Author: jabelone

memberportal/api_general/views.py

@@ -60,6 +60,7 @@ def get(self, request):
                 "senderId": config.SMS_SENDER_ID,
                 "footer": config.SMS_FOOTER,
             },
+            "enableStatsPage": config.ENABLE_STATS_PAGE,
         }
 
         keys = {"stripePublishableKey": config.STRIPE_PUBLISHABLE_KEY}

memberportal/api_member_bucks/urls.py

@@ -22,4 +22,9 @@
         views.MemberBucksDonateFunds.as_view(),
         name="MemberBucksDonateFunds",
     ),
+    path(
+        "api/memberbucks/balance-list/",
+        views.GetMemberbucksBalanceList.as_view(),
+        name="GetMemberbucksBalanceList",
+    ),
 ]

memberportal/api_member_bucks/views.py

@@ -1,3 +1,7 @@
+from django.db.models import Sum
+from rest_framework_api_key.permissions import HasAPIKey
+
+from profile.models import Profile
 from memberbucks.models import MemberBucks
 from rest_framework import status, permissions
 from rest_framework.response import Response
@@ -139,3 +143,28 @@ def post(self, request, amount=None):
             ),
         )
         return Response()
+
+
+class GetMemberbucksBalanceList(APIView):
+    """
+    get: This method returns the balance for every member in the system and the total memberbucks in circulation.
+    """
+
+    permission_classes = (permissions.IsAdminUser | HasAPIKey,)
+
+    def get(self, request):
+        total_balance = Profile.objects.filter(memberbucks_balance__lt=1000).aggregate(
+            Sum("memberbucks_balance")
+        )
+        member_balances = (
+            Profile.objects.all()
+            .order_by("-memberbucks_balance")
+            .values("first_name", "last_name", "screen_name", "memberbucks_balance")
+        )
+        return Response(
+            {
+                "total_memberbucks": total_balance["memberbucks_balance__sum"],
+                "member_balances": member_balances,
+            },
+            status=status.HTTP_200_OK,
+        )

memberportal/api_metrics/metrics.py

@@ -57,7 +57,10 @@ def calculate_member_count():
         profile_states.append({"state": state["state"], "total": state["count"]})
 
     Metric.objects.create(
-        name=Metric.MetricName.MEMBER_COUNT_TOTAL, data=profile_states
+        name=Metric.MetricName.MEMBER_COUNT_TOTAL,
+        data=(
+            profile_states if len(profile_states) else [{"state": "active", "total": 0}]
+        ),
     ).full_clean()
 
 
@@ -74,7 +77,10 @@ def calculate_member_count_6_months():
         profile_states.append({"state": state["state"], "total": state["count"]})
 
     Metric.objects.create(
-        name=Metric.MetricName.MEMBER_COUNT_6_MONTHS, data=profile_states
+        name=Metric.MetricName.MEMBER_COUNT_6_MONTHS,
+        data=(
+            profile_states if len(profile_states) else [{"state": "active", "total": 0}]
+        ),
     ).full_clean()
 
 
@@ -91,7 +97,10 @@ def calculate_member_count_12_months():
         profile_states.append({"state": state["state"], "total": state["count"]})
 
     Metric.objects.create(
-        name=Metric.MetricName.MEMBER_COUNT_12_MONTHS, data=profile_states
+        name=Metric.MetricName.MEMBER_COUNT_12_MONTHS,
+        data=(
+            profile_states if len(profile_states) else [{"state": "active", "total": 0}]
+        ),
     ).full_clean()
 
 
@@ -109,13 +118,19 @@ def calculate_subscription_count():
         )
     Metric.objects.create(
         name=Metric.MetricName.SUBSCRIPTION_COUNT_TOTAL,
-        data=subscription_states_data,
+        data=(
+            subscription_states_data
+            if len(subscription_states_data)
+            else [{"state": "inactive", "total": 0}]
+        ),
     ).full_clean()
 
 
 def calculate_memberbucks_balance():
     logger.debug("Calculating memberbucks balance total")
-    total_balance = Profile.objects.aggregate(Sum("memberbucks_balance"))
+    total_balance = Profile.objects.filter(memberbucks_balance__lt=1000).aggregate(
+        Sum("memberbucks_balance")
+    )
     Metric.objects.create(
         name=Metric.MetricName.MEMBERBUCKS_BALANCE_TOTAL,
         data={"value": total_balance["memberbucks_balance__sum"]},
@@ -127,7 +142,8 @@ def calculate_memberbucks_transactions():
     logger.debug("Calculating subscription count total")
     transaction_data = []
     for transaction_type in (
-        MemberBucks.objects.values("transaction_type")
+        MemberBucks.objects.filter(amount__lt=1000)
+        .values("transaction_type")
         .annotate(amount=Sum("amount"))
         .order_by("-amount")
     ):
@@ -139,5 +155,9 @@ def calculate_memberbucks_transactions():
         )
     Metric.objects.create(
         name=Metric.MetricName.MEMBERBUCKS_TRANSACTIONS_TOTAL,
-        data=transaction_data,
+        data=(
+            transaction_data
+            if len(transaction_data)
+            else [{"type": "stripe", "total": 0.0}]
+        ),
     ).full_clean()

memberportal/api_metrics/tasks.py

@@ -10,12 +10,23 @@
 
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender, **kwargs):
-    sender.add_periodic_task(
-        config.METRICS_INTERVAL,
-        calculate_metrics.s(),
-        expires=60,
-        name="celery_calculate_metrics",
-    )
+    if config.METRICS_INTERVAL:
+        metrics_interval = config.METRICS_INTERVAL
+        if metrics_interval < 3600 * 24:
+            logger.warning(
+                "METRICS_INTERVAL is less than 24 hours. This is NOT recommended for production and has little benefit."
+            )
+        if metrics_interval < 60:
+            logger.warning(
+                "METRICS_INTERVAL is less than 60 seconds, setting to 60 seconds."
+            )
+            metrics_interval = 60
+        sender.add_periodic_task(
+            metrics_interval,
+            calculate_metrics.s(),
+            expires=60,
+            name="celery_calculate_metrics",
+        )
 
 
 @app.task

memberportal/api_metrics/urls.py

@@ -13,4 +13,9 @@
         views.UpdatePromMetrics.as_view(),
         name="api_update_prom_metrics",
     ),
+    path(
+        "api/update-statistics/",
+        views.UpdateStatistics.as_view(),
+        name="api_update_statistics",
+    ),
 ]

memberportal/api_metrics/views.py

@@ -1,7 +1,12 @@
-import api_metrics.metrics
+from django.db.models import Max, Sum
+from django.utils import timezone
+from rest_framework_api_key.permissions import HasAPIKey
+
+import api_metrics.metrics as api_metrics
 from api_metrics.models import Metric
 from api_general.models import SiteSession
 
+from constance import config
 from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework import permissions
@@ -16,17 +21,67 @@ class Statistics(APIView):
     """
 
     def get(self, request):
+        statistics = {}
+
+        # On site members
+        on_site = {"members": [], "count": 0}
         members = SiteSession.objects.filter(signout_date=None).order_by("-signin_date")
-        member_list = []
+        on_site["count"] = members.count()
 
         for member in members:
-            member_list.append(member.user.profile.get_full_name())
-
-        statistics = {"onSite": {"members": member_list, "count": members.count()}}
+            on_site["members"].append(member.user.profile.get_full_name())
+
+        statistics["on_site"] = on_site
+
+        for metric_name in Metric.MetricName.values:
+            # Don't return any data from the API if the stats page isn't enabled
+            if config.ENABLE_STATS_PAGE or request.user.is_admin:
+                metric_data = []
+                one_year_before_today = timezone.now() - timezone.timedelta(
+                    days=config.STATS_MAX_DAYS
+                )
+                last_stat_per_day = (
+                    Metric.objects.filter(
+                        name=metric_name, creation_date__gte=one_year_before_today
+                    )
+                    .extra(select={"the_date": "date(creation_date)"})
+                    .values_list("the_date")
+                    .order_by("-the_date")
+                    .annotate(max_date=Max("creation_date"))
+                )
+                max_dates = [item[1] for item in last_stat_per_day]
+                metrics = Metric.objects.filter(
+                    name=metric_name, creation_date__in=max_dates
+                ).order_by("creation_date")
+                for metric in metrics:
+                    metric_data.append(
+                        {"date": metric.creation_date, "data": metric.data}
+                    )
+                statistics[metric_name] = metric_data
+            else:
+                statistics[metric_name] = []
 
         return Response(statistics)
 
 
+class UpdateStatistics(APIView):
+    """
+    put: This method updates and stores a new set of statistics.
+    """
+
+    permission_classes = (permissions.IsAdminUser | HasAPIKey,)
+
+    def put(self, request):
+        api_metrics.calculate_member_count()
+        api_metrics.calculate_member_count_6_months()
+        api_metrics.calculate_member_count_12_months()
+        api_metrics.calculate_subscription_count()
+        api_metrics.calculate_memberbucks_balance()
+        api_metrics.calculate_memberbucks_transactions()
+
+        return Response()
+
+
 class UpdatePromMetrics(APIView):
     """
     post: triggers Django to update the Prometheus site metrics from the database.

memberportal/membermatters/constance_config.py

@@ -334,6 +334,14 @@
         3600,
         "The interval in seconds to calculate and store application level metrics data like member count and door swipes.",
     ),
+    "ENABLE_STATS_PAGE": (
+        True,
+        "Enable the stats page that shows member counts and other metrics.",
+    ),
+    "STATS_MAX_DAYS": (
+        365,
+        "The maximum number of days to show on the stats page.",
+    ),
 }
 
 CONSTANCE_CONFIG_FIELDSETS = OrderedDict(
@@ -366,6 +374,7 @@
                 "ENABLE_DOOR_BUMP_API",
             ),
         ),
+        ("Stats Settings", ("ENABLE_STATS_PAGE", "STATS_MAX_DAYS")),
         (
             "Sentry Error Reporting",
             (