test: add loadtest tracking memory/cpu usage

Ensures performance is maintained.

We run call wait_for_many_gets() (runs 10K GET requests) in a Nix wrapper script, net-loadtest,
which measures the CPU and memory each second using psrecord. net-loadtest can be used locally too.

psrecord can also generate a nice png file, but AFAICT there's no way to nicely render this on Github CI.
The png can be attached as an artifact, but you have to download it and open it (too slow for feedback).

Author: steve-chavez

.github/workflows/main.yml

@@ -17,7 +17,7 @@ jobs:
         uses: cachix/install-nix-action@v30
 
       - name: Use Cachix Cache
-        uses: cachix/cachix-action@v10
+        uses: cachix/cachix-action@v16
         with:
           name: nxpg
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
@@ -45,7 +45,7 @@ jobs:
           #nix_path: nixpkgs=channel:nixos-unstable
 
       #- name: Use Cachix Cache
-        #uses: cachix/cachix-action@v10
+        #uses: cachix/cachix-action@v16
         #with:
           #name: xpg
           #authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
@@ -56,6 +56,28 @@ jobs:
       #- name: Run tests
         #run: nix-shell --run "nxpg -v ${{ matrix.pg-version }} test"
 
+  loadtest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v30
+
+      - name: Use Cachix Cache
+        uses: cachix/cachix-action@v16
+        with:
+          name: nxpg
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - name: Build
+        run: nix-shell --run "xpg build"
+
+      - name: Run load test
+        run: |
+          nix-shell --run "net-loadtest"
+          cat psrecord.md >> "$GITHUB_STEP_SUMMARY"
+
   coverage:
 
     runs-on: ubuntu-latest
@@ -71,7 +93,7 @@ jobs:
         uses: cachix/install-nix-action@v30
 
       - name: Use Cachix Cache
-        uses: cachix/cachix-action@v10
+        uses: cachix/cachix-action@v16
         with:
           name: nxpg
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}

.gitignore

@@ -20,3 +20,4 @@ nginx.pid
 tags
 net_worker.pid
 sql/pg_net--*.sql
+psrecord.*

nix/loadtest.nix

@@ -0,0 +1,56 @@
+{ writeShellScriptBin, psrecord, writers, python3Packages } :
+
+let
+  toMarkdown =
+    writers.writePython3 "psrecord-to-md"
+      {
+        libraries = [ python3Packages.pandas python3Packages.tabulate ];
+      }
+      ''
+        import sys
+        import pandas as pd
+        import re
+
+        HEADER_SPLIT = re.compile(r"\s{2,}")
+
+        raw_lines = sys.stdin.read().splitlines()
+
+        header_line = next(
+            (line for line in raw_lines if line.lstrip().startswith("#")), None
+        )
+        if header_line is None:
+            sys.exit("Error: no header line found in input.")
+
+        columns = HEADER_SPLIT.split(header_line.lstrip("#").strip())
+
+        data_lines = [
+            line.strip()
+            for line in raw_lines
+            if line.strip() and not line.lstrip().startswith("#")
+        ]
+
+        data_rows = [HEADER_SPLIT.split(line) for line in data_lines]
+
+        df = pd.DataFrame(data_rows, columns=columns, dtype=str)
+
+        df.to_markdown(sys.stdout, index=False, tablefmt="github")
+      '';
+in
+
+writeShellScriptBin "net-loadtest" ''
+  set -euo pipefail
+
+  net-with-nginx xpg psql -c "call wait_for_many_gets()" > /dev/null &
+
+  # wait for process to start so we can capture it with psrecord
+  sleep 2
+
+  record_log=psrecord.log
+  record_result=psrecord.md
+
+  ${psrecord}/bin/psrecord $(cat build-17/bgworker.pid) --interval 1 --log "$record_log" > /dev/null
+
+  cat $record_log | ${toMarkdown} > $record_result
+
+  echo "generated $record_result"
+''

nix/nixops.nix

@@ -114,7 +114,7 @@ in {
       };
       initialScript = pkgs.writeText "init-sql-script" ''
         create extension pg_net;
-        ${builtins.readFile ./bench.sql}
+        ${builtins.readFile ../test/loadtest/loadtest.sql}
       '';
     };
 
@@ -163,10 +163,10 @@ in {
         ''
       )
       (
-        pkgs.writeShellScriptBin "psql-reproduce-timeouts" ''
+        pkgs.writeShellScriptBin "psql-net-many-gets" ''
           set -euo pipefail
 
-          psql -U postgres -c "call repro_timeouts();"
+          psql -U postgres -c "call wait_for_many_gets(url:='$1');"
         ''
       )
     ];

nix/xpg.nix

@@ -3,8 +3,8 @@ let
   dep = fetchFromGitHub {
     owner  = "steve-chavez";
     repo   = "xpg";
-    rev    = "v1.3.2";
-    sha256 = "sha256-ooYqMOQD9y+/87wBd33Mvbpsx+FwEMdZoibGRM4gvBk=";
+    rev    = "v1.3.3";
+    sha256 = "sha256-N0/jp+tOWFz72Z6ZK2oA0M6e5F1W3SPhgk5G/0PbBso=";
   };
   xpg = import dep;
 in

shell.nix

@@ -9,6 +9,7 @@ mkShell {
       nginxCustom = callPackage ./nix/nginxCustom.nix {};
       nixopsScripts = callPackage ./nix/nixopsScripts.nix {};
       xpg = callPackage ./nix/xpg.nix {inherit fetchFromGitHub;};
+      loadtest = callPackage ./nix/loadtest.nix {};
       pythonDeps = with python3Packages; [
         pytest
         psycopg2
@@ -20,6 +21,7 @@ mkShell {
       pythonDeps
       nginxCustom.nginxScript
       curl
+      loadtest
     ] ++
     nixopsScripts;
   shellHook = ''

test/init.sql

@@ -1,4 +1,4 @@
 create database pre_existing;
 create role pre_existing nosuperuser login;
 create extension pg_net;
-\ir ../nix/bench.sql
+\ir ./loadtest/loadtest.sql

nix/bench.sql renamed to test/loadtest/loadtest.sql

@@ -5,7 +5,8 @@ select
   (select error_msg from net._http_response where error_msg is not null order by id desc limit 1) as last_failure_error
 from net._http_response;
 
-create or replace procedure repro_timeouts(number_of_requests int default 10000, url text default 'http://server') as $$
+-- loadtest using many gets, used to be called `repro_timeouts`
+create or replace procedure wait_for_many_gets(number_of_requests int default 10000, url text default 'http://localhost:8080') as $$
 declare
   last_id bigint;
   first_time timestamptz;