show system wide code issue/claim chart (#2372)

mikehelmick · web-flow · commit 92a785d03510 · 2022-08-26T18:23:34.000Z
* show system wide code issue/claim chart

* add missing return

* enforce min number of realms contributing to a stats day

* enforce min min

* lint
diff --git a/assets/server/admin/realms/index.html b/assets/server/admin/realms/index.html
@@ -8,6 +8,7 @@
 <html dir="{{$.textDirection}}" lang="{{$.textLanguage}}">
 <head>
   {{template "head" .}}
+  <script defer src="https://www.gstatic.com/charts/loader.js"></script>
 </head>
 
 <body id="admin-realms-index" class="tab-content">
@@ -16,6 +17,22 @@
   <main role="main" class="container">
     {{template "flash" .}}
 
+    <div class="card shadow-sm mb-3">
+      <div class="card-header">
+        <i class="bi bi-graph-up me-2"></i>
+        System wide verification code statistics
+      </div>
+      <div id="dashboard_div">
+        <div id="system_chart_div" class="h-100 w-100" style="min-height:325px;">
+          <p class="text-center font-italic w-100 mt-5">Loading chart...</p>
+        </div>
+        <div id="filter_div" class="text-end" style="height: 75px;"></div>
+      </div>
+      <small class="card-footer d-flex justify-content-between text-muted">
+        <p>An individual day requires data from at least <em>{{.minRealms}}</em> realms to be shown.</p>
+      </small>
+    </div>
+
     <div class="card mb-3 shadow-sm">
       <div class="card-header">
         <i class="bi bi-house-door me-2"></i>
diff --git a/assets/server/static/js/admin-stats.js b/assets/server/static/js/admin-stats.js
@@ -0,0 +1,159 @@
+(() => {
+    window.addEventListener('load', async (event) => {
+      const containerChart = document.querySelector('div#system_chart_div');
+      if (!containerChart) {
+        return;
+      }
+  
+      google.charts.load('current', {
+        packages: ['corechart', 'controls'],
+        callback: drawCharts,
+      });
+  
+      function drawCharts() {
+        const request = new XMLHttpRequest();
+        request.open('GET', '/admin/stats/system.json');
+        request.overrideMimeType('application/json');
+  
+        request.onload = (event) => {
+          const data = JSON.parse(request.response);
+          drawSystemCodeChart(data);
+        };
+  
+        request.onerror = (event) => {
+          console.error('error from response: ' + request.response);
+          flash.error('Failed to load realm stats: ' + err);
+        };
+  
+        request.send();
+      }
+  
+      function drawSystemCodeChart(data) {
+        const charts = [
+          {
+            chartType: 'LineChart',
+            chartDiv: '#system_chart_div',
+            dashboardDiv: '#dashboard_div',
+            filterDiv: '#filter_div',
+            headerFunc: (dataTable, hasKeyServerStats) => {
+              dataTable.addColumn('date', 'Date');
+              dataTable.addColumn('number', 'Codes Issued');
+              dataTable.addColumn('number', 'Codes Claimed');
+            },
+            rowFunc: (dataTable, row, hasKeyServerStats) => {
+              if (hasKeyServerStats) {
+                dataTable.addRow([
+                  utcDate(row.date),
+                  row.data.codes_issued,
+                  row.data.codes_claimed,
+                ]);
+              } else {
+                dataTable.addRow([
+                  utcDate(row.date),
+                  row.data.codes_issued,
+                  row.data.codes_claimed,
+                ]);
+              }
+            },
+          },     
+        ];
+  
+        const dateFormatter = new google.visualization.DateFormat({
+          pattern: 'MMM dd',
+        });
+  
+        for (let i = 0; i < charts.length; i++) {
+          const chart = charts[i];
+          const chartContainer = document.querySelector(chart.chartDiv);
+          const dashboardContainer = document.querySelector(chart.dashboardDiv);
+          const filterContainer = document.querySelector(chart.filterDiv);
+  
+          if (!chartContainer || !dashboardContainer || !filterContainer) {
+            continue;
+          }
+  
+          if (!data || !data.statistics) {
+            const pContainer = chartContainer.querySelector('p');
+            pContainer.innerText = 'No data yet.';
+            continue;
+          }
+  
+          const hasKeyServerStats = data.has_key_server_stats;
+          const win = Math.min(30, data.statistics.length - 1);
+          const startChart = new Date(data.statistics[win].date);
+  
+          const dataTable = new google.visualization.DataTable();
+          chart.headerFunc(dataTable, hasKeyServerStats);
+          for (let j = 0; j < data.statistics.length; j++) {
+            const stat = data.statistics[j];
+            chart.rowFunc(dataTable, stat, hasKeyServerStats);
+          }
+          dateFormatter.format(dataTable, 0);
+  
+          const dashboard = new google.visualization.Dashboard(dashboardContainer);
+          const filter = new google.visualization.ControlWrapper({
+            controlType: 'ChartRangeFilter',
+            containerId: filterContainer,
+            state: {
+              range: {
+                start: startChart,
+              },
+            },
+            options: {
+              filterColumnIndex: 0,
+              series: {
+                0: {
+                  opacity: 0,
+                },
+              },
+              ui: {
+                chartType: 'LineChart',
+                chartOptions: {
+                  colors: ['#dddddd'],
+                  chartArea: {
+                    width: '100%',
+                    height: '100%',
+                    top: 0,
+                    right: 40,
+                    bottom: 20,
+                    left: 60,
+                  },
+                  isStacked: true,
+                  hAxis: { format: 'M/d' },
+                },
+                chartView: {
+                  columns: [0, 1],
+                },
+                minRangeSize: 86400000, // ms for 1 day
+              },
+            },
+          });
+  
+          const realmChart = new google.visualization.ChartWrapper({
+            chartType: chart.chartType,
+            containerId: chartContainer,
+            options: {
+              colors: ['#007bff', '#28a745', '#dc3545', '#6c757d', '#ee8c00'],
+              chartArea: {
+                left: 60,
+                right: 40,
+                bottom: 5,
+                top: 40,
+                width: '100%',
+                height: '300',
+              },
+              isStacked: true,
+              hAxis: { textPosition: 'none' },
+              legend: { position: 'top' },
+              width: '100%',
+            },
+          });
+  
+          dashboard.bind(filter, realmChart);
+          dashboard.draw(dataTable);
+          debounce('resize', async () => dashboard.draw(dataTable));
+        }
+      }
+    });
+  })();
+  
diff --git a/internal/routes/server.go b/internal/routes/server.go
@@ -514,6 +514,8 @@ func systemAdminRoutes(r *mux.Router, c *admin.Controller) {
 	r.Handle("", http.RedirectHandler("/admin/realms", http.StatusSeeOther)).Methods(http.MethodGet)
 	r.Handle("/", http.RedirectHandler("/admin/realms", http.StatusSeeOther)).Methods(http.MethodGet)
 
+	r.Handle("/stats/system.json", c.HandleCodeStats()).Methods(http.MethodGet)
+
 	r.Handle("/realms", c.HandleRealmsIndex()).Methods(http.MethodGet)
 	r.Handle("/realms", c.HandleRealmsCreate()).Methods(http.MethodPost)
 	r.Handle("/realms/new", c.HandleRealmsCreate()).Methods(http.MethodGet)
diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go
@@ -100,6 +100,10 @@ type ServerConfig struct {
 	// If MaintenanceMode is true, the server is temporarily read-only and will not issue codes.
 	MaintenanceMode bool `env:"MAINTENANCE_MODE"`
 
+	// MinRealmsForSystemStatistics gives a minimum threshold for displaying system
+	// admin level statistics
+	MinRealmsForSystemStatistics uint `env:"MIN_REALMS_FOR_SYSTEM_STATS, default=2"`
+
 	// Rate limiting configuration
 	RateLimit ratelimit.Config
 }
@@ -165,6 +169,10 @@ func (c *ServerConfig) Validate() error {
 		return fmt.Errorf("failed to validate issue API configuration: %w", err)
 	}
 
+	if c.MinRealmsForSystemStatistics < 2 {
+		return fmt.Errorf("MIN_REALMS_FOR_SYSTEM_STATS cannot be set lower than 2")
+	}
+
 	return nil
 }
 
diff --git a/pkg/controller/admin/code_stats.go b/pkg/controller/admin/code_stats.go
@@ -0,0 +1,37 @@
+// Copyright 2022 the Exposure Notifications Verification Server authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package admin
+
+import (
+	"net/http"
+
+	"github.com/google/exposure-notifications-verification-server/pkg/controller"
+)
+
+// HandleCodeStats returns the code issue / claimed
+// rates across all realms.
+func (c *Controller) HandleCodeStats() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+
+		stats, err := c.db.AllRealmCodeStatsCached(ctx, c.cacher, int(c.config.MinRealmsForSystemStatistics))
+		if err != nil {
+			controller.InternalError(w, r, c.h, err)
+			return
+		}
+
+		c.h.RenderJSON(w, http.StatusOK, stats)
+	})
+}
diff --git a/pkg/controller/admin/realms.go b/pkg/controller/admin/realms.go
@@ -69,6 +69,7 @@ func (c *Controller) HandleRealmsIndex() http.Handler {
 		m["memberships"] = membershipsMap
 		m["query"] = q
 		m["paginator"] = paginator
+		m["minRealms"] = c.config.MinRealmsForSystemStatistics
 		c.h.RenderHTML(w, "admin/realms/index", m)
 	})
 }
diff --git a/pkg/database/realm.go b/pkg/database/realm.go
@@ -1922,6 +1922,79 @@ func (r *Realm) Stats(db *Database) (RealmStats, error) {
 	return stats, nil
 }
 
+// AllRealmCodeStatsCached returns combined code issue / claimed stats for all realms
+// in the system, but cached.
+func (db *Database) AllRealmCodeStatsCached(ctx context.Context, cacher cache.Cacher, requiredRealms int) (RealmStats, error) {
+	if cacher == nil {
+		return nil, fmt.Errorf("cacher cannot be nil")
+	}
+
+	var stats RealmStats
+	cacheKey := &cache.Key{
+		Namespace: "stats:system",
+		Key:       fmt.Sprintf("all:min-%d", requiredRealms),
+	}
+	if err := cacher.Fetch(ctx, cacheKey, &stats, 30*time.Minute, func() (interface{}, error) {
+		return db.AllRealmCodeStats(ctx, requiredRealms)
+	}); err != nil {
+		return nil, err
+	}
+
+	return stats, nil
+}
+
+// AllRealmCodeStats returns the usage statistics for all realms.
+// This reuses the RealmStats data structure and the realm_id is the COUNT
+// of all realms that contributed to statistics on that day, so that filtering
+// can be done if there are not enough data points.
+func (db *Database) AllRealmCodeStats(ctx context.Context, requiredRealms int) (RealmStats, error) {
+	stop := timeutils.UTCMidnight(time.Now())
+	start := stop.Add(project.StatsDisplayDays * -24 * time.Hour)
+	if start.After(stop) {
+		return nil, ErrBadDateRange
+	}
+
+	sql := `
+		SELECT
+			d.date AS date,
+			COUNT(DISTINCT(realm_id)) as realm_id,
+			SUM(s.codes_issued) AS codes_issued,		
+			SUM(s.codes_claimed) AS codes_claimed,
+			0 AS codes_invalid,
+			0 AS user_reports_issued,
+			0 AS user_reports_claimed,
+			0 AS tokens_claimed,
+			0 AS tokens_invalid,
+			0 AS user_report_tokens_claimed,
+			array[]::integer[] AS code_claim_age_distribution,
+			0 AS code_claim_mean_age,
+			array[0,0,0]::bigint[] AS codes_invalid_by_os,
+			0 AS user_reports_invalid_nonce
+		FROM (
+			SELECT date::date FROM generate_series($1, $2, '1 day'::interval) date
+		) d
+		LEFT JOIN realm_stats s ON s.date = d.date
+		GROUP BY d.date
+		ORDER BY date DESC`
+
+	var stats []*RealmStat
+	if err := db.db.Raw(sql, start, stop).Scan(&stats).Error; err != nil {
+		if IsNotFound(err) {
+			return stats, nil
+		}
+		return nil, err
+	}
+
+	for _, s := range stats {
+		if s.RealmID < uint(requiredRealms) {
+			s.CodesIssued = 0
+			s.CodesClaimed = 0
+		}
+	}
+
+	return stats, nil
+}
+
 // StatsCached is stats, but cached.
 func (r *Realm) StatsCached(ctx context.Context, db *Database, cacher cache.Cacher) (RealmStats, error) {
 	if cacher == nil {
diff --git a/tools/seed/main.go b/tools/seed/main.go

Original file line number	Diff line number	Diff line change
`@@ -100,6 +100,10 @@ type ServerConfig struct {`
`100`	`100`	`// If MaintenanceMode is true, the server is temporarily read-only and will not issue codes.`
`101`	`101`	MaintenanceMode bool `env:"MAINTENANCE_MODE"`
`102`	`102`
	`103`	`+ // MinRealmsForSystemStatistics gives a minimum threshold for displaying system`
	`104`	`+ // admin level statistics`
	`105`	+ MinRealmsForSystemStatistics uint `env:"MIN_REALMS_FOR_SYSTEM_STATS, default=2"`
	`106`	`+`
`103`	`107`	`// Rate limiting configuration`
`104`	`108`	`RateLimit ratelimit.Config`
`105`	`109`	`}`
`@@ -165,6 +169,10 @@ func (c *ServerConfig) Validate() error {`
`165`	`169`	`return fmt.Errorf("failed to validate issue API configuration: %w", err)`
`166`	`170`	`}`
`167`	`171`
	`172`	`+ if c.MinRealmsForSystemStatistics < 2 {`
	`173`	`+ return fmt.Errorf("MIN_REALMS_FOR_SYSTEM_STATS cannot be set lower than 2")`
	`174`	`+ }`
	`175`	`+`
`168`	`176`	`return nil`
`169`	`177`	`}`
`170`	`178`
Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,7 @@ func (c *Controller) HandleRealmsIndex() http.Handler {`
`69`	`69`	`m["memberships"] = membershipsMap`
`70`	`70`	`m["query"] = q`
`71`	`71`	`m["paginator"] = paginator`
	`72`	`+ m["minRealms"] = c.config.MinRealmsForSystemStatistics`
`72`	`73`	`c.h.RenderHTML(w, "admin/realms/index", m)`
`73`	`74`	`})`
`74`	`75`	`}`