Implement "Year over Year" comparison mode (#2704)

vinibrsl · ukutaht · web-flow · commit 8b0f0cabc211 · 2023-03-07T12:52:26.000-03:00
This pull request adds support for multiple comparison modes, changes the comparison checkbox to a combobox, and implements the year over year comparison mode. The feature is still behind a feature flag.

Co-authored-by: Uku Taht &lt;uku.taht@gmail.com&gt;
diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.js
@@ -45,7 +45,7 @@ export function serializeQuery(query, extraQuery=[]) {
   if (query.filters) { queryObj.filters = serializeFilters(query.filters)  }
   if (query.with_imported) { queryObj.with_imported = query.with_imported  }
   if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH }
-  if (query.comparison) { queryObj.comparison = true }
+  if (query.comparison) { queryObj.comparison = query.comparison }
   Object.assign(queryObj, ...extraQuery)
 
   return '?' + serialize(queryObj)
diff --git a/assets/js/dashboard/comparison-input.js b/assets/js/dashboard/comparison-input.js
@@ -1,21 +1,61 @@
-import React from 'react'
+import React, { Fragment } from 'react'
 import { withRouter } from "react-router-dom";
 import { navigateToQuery } from './query'
+import { Menu, Transition } from '@headlessui/react'
+import { ChevronDownIcon } from '@heroicons/react/20/solid'
+import classNames from 'classnames'
+
+const COMPARISON_MODES = {
+  'previous_period': 'Previous period',
+  'year_over_year': 'Year over year',
+}
 
 export const COMPARISON_DISABLED_PERIODS = ['realtime', 'all']
 
 const ComparisonInput = function({ site, query, history }) {
   if (!site.flags.comparisons) return null
   if (COMPARISON_DISABLED_PERIODS.includes(query.period)) return null
 
-  function update(event) {
-    navigateToQuery(history, query, { comparison: event.target.checked })
+  function update(key) {
+    navigateToQuery(history, query, { comparison: key })
+  }
+
+  function renderItem({ label, value, isCurrentlySelected }) {
+    const labelClass = classNames("font-medium text-sm", { "font-bold disabled": isCurrentlySelected })
+
+    return (
+      <Menu.Item
+        key={value}
+        onClick={() => update(value)}
+        className="px-4 py-2 leading-tight hover:bg-gray-100 dark:text-white hover:text-gray-900 dark:hover:bg-gray-900 dark:hover:text-gray-100 flex hover:cursor-pointer">
+        <span className={labelClass}>{ label }</span>
+      </Menu.Item>
+    )
   }
 
   return (
-    <div className="flex-none mx-3">
-      <input id="comparison-input" type="checkbox" onChange={update} checked={query.comparison} className="h-4 w-4 rounded border-gray-300 text-indigo-600 focus:ring-indigo-500" />
-      <label htmlFor="comparison-input" className="ml-1.5 font-medium text-xs md:text-sm text-gray-700 dark:text-white">Compare</label>
+    <div className="flex ml-auto pl-2">
+    <div className="w-20 sm:w-36 md:w-48 md:relative">
+      <Menu as="div" className="relative inline-block pl-2 w-full">
+        <Menu.Button className="bg-white text-gray-800 text-xs md:text-sm font-medium dark:bg-gray-800 dark:hover:bg-gray-900 dark:text-gray-200 hover:bg-gray-200 flex md:px-3 px-2 py-2 items-center justify-between leading-tight rounded shadow truncate cursor-pointer w-full">
+          <span>{ COMPARISON_MODES[query.comparison] || 'Compare to' }</span>
+          <ChevronDownIcon className="hidden sm:inline-block h-4 w-4 md:h-5 md:w-5 text-gray-500 ml-5" />
+        </Menu.Button>
+        <Transition
+          as={Fragment}
+          enter="transition ease-out duration-100"
+          enterFrom="transform opacity-0 scale-95"
+          enterTo="transform opacity-100 scale-100"
+          leave="transition ease-in duration-75"
+          leaveFrom="transform opacity-100 scale-100"
+          leaveTo="transform opacity-0 scale-95">
+          <Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static>
+            { renderItem({ label: "Disabled", value: false, isCurrentlySelected: !query.comparison }) }
+            { Object.keys(COMPARISON_MODES).map((key) => renderItem({ label: COMPARISON_MODES[key], value: key, isCurrentlySelected: key == query.comparison})) }
+          </Menu.Items>
+        </Transition>
+      </Menu>
+    </div>
     </div>
   )
 }
diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js
@@ -19,7 +19,7 @@ export function parseQuery(querystring, site) {
     period = '30d'
   }
 
-  let comparison = !!q.get('comparison')
+  let comparison = q.get('comparison')
   if (COMPARISON_DISABLED_PERIODS.includes(period)) comparison = null
 
   return {
diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex
@@ -0,0 +1,70 @@
+defmodule Plausible.Stats.Comparisons do
+  @moduledoc """
+  This module provides functions for comparing query periods.
+
+  It allows you to compare a given period with a previous period or with the
+  same period from the previous year. For example, you can compare this month's
+  main graph with last month or with the same month from last year.
+  """
+
+  alias Plausible.Stats
+
+  @modes ~w(previous_period year_over_year)
+  @disallowed_periods ~w(realtime all)
+
+  @type mode() :: String.t() | nil
+
+  @spec compare(
+          Plausible.Site.t(),
+          Stats.Query.t(),
+          mode(),
+          NaiveDateTime.t() | nil
+        ) :: {:ok, Stats.Query.t()} | {:error, :not_supported}
+  def compare(
+        %Plausible.Site{} = site,
+        %Stats.Query{} = source_query,
+        mode,
+        now \\ nil
+      ) do
+    if valid_mode?(source_query, mode) do
+      now = now || Timex.now(site.timezone)
+      {:ok, do_compare(source_query, mode, now)}
+    else
+      {:error, :not_supported}
+    end
+  end
+
+  defp do_compare(source_query, "year_over_year", now) do
+    start_date = Date.add(source_query.date_range.first, -365)
+    end_date = earliest(source_query.date_range.last, now) |> Date.add(-365)
+
+    range = Date.range(start_date, end_date)
+    %Stats.Query{source_query | date_range: range}
+  end
+
+  defp do_compare(source_query, "previous_period", now) do
+    last = earliest(source_query.date_range.last, now)
+    diff_in_days = Date.diff(source_query.date_range.first, last) - 1
+
+    new_first = Date.add(source_query.date_range.first, diff_in_days)
+    new_last = Date.add(last, diff_in_days)
+
+    range = Date.range(new_first, new_last)
+    %Stats.Query{source_query | date_range: range}
+  end
+
+  defp earliest(a, b) do
+    if Date.compare(a, b) in [:eq, :lt], do: a, else: b
+  end
+
+  @spec valid_mode?(Stats.Query.t(), mode()) :: boolean()
+  @doc """
+  Returns whether the source query and the selected mode support comparisons.
+
+  For example, the realtime view doesn't support comparisons. Additionally, only
+  #{inspect(@modes)} are supported.
+  """
+  def valid_mode?(%Stats.Query{period: period}, mode) do
+    mode in @modes && period not in @disallowed_periods
+  end
+end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
@@ -10,58 +10,7 @@ defmodule Plausible.Stats.Query do
   require OpenTelemetry.Tracer, as: Tracer
   alias Plausible.Stats.{FilterParser, Interval}
 
-  def shift_back(%__MODULE__{period: "year"} = query, site) do
-    # Querying current year to date
-    {new_first, new_last} =
-      if Timex.compare(Timex.now(site.timezone), query.date_range.first, :year) == 0 do
-        diff =
-          Timex.diff(
-            Timex.beginning_of_year(Timex.now(site.timezone)),
-            Timex.now(site.timezone),
-            :days
-          ) - 1
-
-        {query.date_range.first |> Timex.shift(days: diff),
-         Timex.now(site.timezone) |> Timex.to_date() |> Timex.shift(days: diff)}
-      else
-        diff = Timex.diff(query.date_range.first, query.date_range.last, :days) - 1
-
-        {query.date_range.first |> Timex.shift(days: diff),
-         query.date_range.last |> Timex.shift(days: diff)}
-      end
-
-    Map.put(query, :date_range, Date.range(new_first, new_last))
-  end
-
-  def shift_back(%__MODULE__{period: "month"} = query, site) do
-    # Querying current month to date
-    {new_first, new_last} =
-      if Timex.compare(Timex.now(site.timezone), query.date_range.first, :month) == 0 do
-        diff =
-          Timex.diff(
-            Timex.beginning_of_month(Timex.now(site.timezone)),
-            Timex.now(site.timezone),
-            :days
-          ) - 1
-
-        {query.date_range.first |> Timex.shift(days: diff),
-         Timex.now(site.timezone) |> Timex.to_date() |> Timex.shift(days: diff)}
-      else
-        diff = Timex.diff(query.date_range.first, query.date_range.last, :days) - 1
-
-        {query.date_range.first |> Timex.shift(days: diff),
-         query.date_range.last |> Timex.shift(days: diff)}
-      end
-
-    Map.put(query, :date_range, Date.range(new_first, new_last))
-  end
-
-  def shift_back(query, _site) do
-    diff = Timex.diff(query.date_range.first, query.date_range.last, :days) - 1
-    new_first = query.date_range.first |> Timex.shift(days: diff)
-    new_last = query.date_range.last |> Timex.shift(days: diff)
-    Map.put(query, :date_range, Date.range(new_first, new_last))
-  end
+  @type t :: %__MODULE__{}
 
   def from(site, %{"period" => "realtime"} = params) do
     date = today(site.timezone)
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
@@ -4,6 +4,10 @@ defmodule Plausible.Stats.Timeseries do
   import Plausible.Stats.{Base, Util}
   use Plausible.Stats.Fragments
 
+  @typep metric :: :pageviews | :visitors | :visits | :bounce_rate | :visit_duration
+  @typep value :: nil | integer() | float()
+  @type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()})
+
   @event_metrics [:visitors, :pageviews]
   @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit]
   def timeseries(site, query, metrics) do
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -20,7 +20,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
          {:ok, metrics} <- parse_and_validate_metrics(params, nil, query) do
       results =
         if params["compare"] == "previous_period" do
-          prev_query = Query.shift_back(query, site)
+          {:ok, prev_query} = Plausible.Stats.Comparisons.compare(site, query, "previous_period")
 
           [prev_result, curr_result] =
             Plausible.ClickhouseRepo.parallel_tasks([
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController do
   use Plausible.Repo
   use Plug.ErrorHandler
   alias Plausible.Stats
-  alias Plausible.Stats.{Query, Filters}
+  alias Plausible.Stats.{Query, Filters, Comparisons}
 
   require Logger
 
@@ -115,9 +115,9 @@ defmodule PlausibleWeb.Api.StatsController do
       full_intervals = build_full_intervals(query, labels)
 
       comparison_result =
-        if params["comparison"] do
-          comparison_query = Query.shift_back(query, site)
-          Stats.timeseries(site, comparison_query, [selected_metric])
+        case Comparisons.compare(site, query, params["comparison"]) do
+          {:ok, comparison_query} -> Stats.timeseries(site, comparison_query, [selected_metric])
+          {:error, :not_supported} -> nil
         end
 
       json(conn, %{
@@ -174,9 +174,10 @@ defmodule PlausibleWeb.Api.StatsController do
     site = conn.assigns[:site]
 
     with :ok <- validate_params(params) do
+      comparison_mode = params["comparison"] || "previous_period"
       query = Query.from(site, params) |> Filters.add_prefix()
 
-      {top_stats, sample_percent} = fetch_top_stats(site, query)
+      {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_mode)
 
       json(conn, %{
         top_stats: top_stats,
@@ -243,7 +244,8 @@ defmodule PlausibleWeb.Api.StatsController do
 
   defp fetch_top_stats(
          site,
-         %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query
+         %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query,
+         _comparison_mode
        ) do
     query_30m = %Query{query | period: "30m"}
 
@@ -270,7 +272,7 @@ defmodule PlausibleWeb.Api.StatsController do
     {stats, 100}
   end
 
-  defp fetch_top_stats(site, %Query{period: "realtime"} = query) do
+  defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_mode) do
     query_30m = %Query{query | period: "30m"}
 
     %{
@@ -296,29 +298,41 @@ defmodule PlausibleWeb.Api.StatsController do
     {stats, 100}
   end
 
-  defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _goal}} = query) do
+  defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _goal}} = query, comparison_mode) do
     total_q = Query.remove_event_filters(query, [:goal, :props])
-    prev_query = Query.shift_back(query, site)
-    prev_total_query = Query.shift_back(total_q, site)
+
+    {prev_converted_visitors, prev_completions} =
+      case Stats.Comparisons.compare(site, query, comparison_mode) do
+        {:ok, prev_query} ->
+          %{visitors: %{value: prev_converted_visitors}, events: %{value: prev_completions}} =
+            Stats.aggregate(site, prev_query, [:visitors, :events])
+
+          {prev_converted_visitors, prev_completions}
+
+        {:error, :not_supported} ->
+          {nil, nil}
+      end
+
+    prev_unique_visitors =
+      case Stats.Comparisons.compare(site, total_q, comparison_mode) do
+        {:ok, prev_total_query} ->
+          site
+          |> Stats.aggregate(prev_total_query, [:visitors])
+          |> get_in([:visitors, :value])
+
+        {:error, :not_supported} ->
+          nil
+      end
 
     %{
       visitors: %{value: unique_visitors}
     } = Stats.aggregate(site, total_q, [:visitors])
 
-    %{
-      visitors: %{value: prev_unique_visitors}
-    } = Stats.aggregate(site, prev_total_query, [:visitors])
-
     %{
       visitors: %{value: converted_visitors},
       events: %{value: completions}
     } = Stats.aggregate(site, query, [:visitors, :events])
 
-    %{
-      visitors: %{value: prev_converted_visitors},
-      events: %{value: prev_completions}
-    } = Stats.aggregate(site, prev_query, [:visitors, :events])
-
     conversion_rate = calculate_cr(unique_visitors, converted_visitors)
     prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors)
 
@@ -348,9 +362,7 @@ defmodule PlausibleWeb.Api.StatsController do
     {stats, 100}
   end
 
-  defp fetch_top_stats(site, query) do
-    prev_query = Query.shift_back(query, site)
-
+  defp fetch_top_stats(site, query, comparison_mode) do
     metrics =
       if query.filters["event:page"] do
         [
@@ -375,7 +387,12 @@ defmodule PlausibleWeb.Api.StatsController do
       end
 
     current_results = Stats.aggregate(site, query, metrics)
-    prev_results = Stats.aggregate(site, prev_query, metrics)
+
+    prev_results =
+      case Stats.Comparisons.compare(site, query, comparison_mode) do
+        {:ok, prev_results_query} -> Stats.aggregate(site, prev_results_query, metrics)
+        {:error, :not_supported} -> nil
+      end
 
     stats =
       [
@@ -394,11 +411,11 @@ defmodule PlausibleWeb.Api.StatsController do
 
   defp top_stats_entry(current_results, prev_results, name, key) do
     if current_results[key] do
-      %{
-        name: name,
-        value: current_results[key][:value],
-        change: calculate_change(key, prev_results[key][:value], current_results[key][:value])
-      }
+      value = get_in(current_results, [key, :value])
+      prev_value = get_in(prev_results, [key, :value])
+      change = prev_value && calculate_change(key, prev_value, value)
+
+      %{name: name, value: value, change: change}
     end
   end
 
diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex
@@ -49,7 +49,7 @@ defmodule Plausible.Workers.SendEmailReport do
   end
 
   defp send_report(email, site, name, unsubscribe_link, query) do
-    prev_query = Query.shift_back(query, site)
+    {:ok, prev_query} = Stats.Comparisons.compare(site, query, "previous_period")
     curr_period = Stats.aggregate(site, query, [:pageviews, :visitors, :bounce_rate])
     prev_period = Stats.aggregate(site, prev_query, [:pageviews, :visitors, :bounce_rate])
 
diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ export function parseQuery(querystring, site) {`
`19`	`19`	`period = '30d'`
`20`	`20`	`}`
`21`	`21`
`22`		`- let comparison = !!q.get('comparison')`
	`22`	`+ let comparison = q.get('comparison')`
`23`	`23`	`if (COMPARISON_DISABLED_PERIODS.includes(period)) comparison = null`
`24`	`24`
`25`	`25`	`return {`