use tsweb debugger (#2420)

This PR switches the homegrown debug endpoint to using tsweb.Debugger, a neat toolkit with batteries included for pprof and friends, and making it easy to add additional debug info:

I've started out by adding a bunch of "introspect" endpoints
image

So users can see the acl, filter, config, derpmap and connected nodes as headscale sees them.

Author: kradalby

CHANGELOG.md

@@ -2,12 +2,15 @@
 
 ## Next
 
-
 ### Changes
 
 - `oidc.map_legacy_users` and `oidc.strip_email_domain` has been removed
   [#2411](https://github.com/juanfont/headscale/pull/2411)
-
+- Add more information to `/debug` endpoint
+  [#2420](https://github.com/juanfont/headscale/pull/2420)
+  - It is now possible to inspect running goroutines and take profiles
+  - View of config, policy, filter, ssh policy per node, connected nodes and
+    DERPmap
 
 ## 0.25.0 (2025-02-xx)
 
@@ -23,7 +26,7 @@
   - A logged out node logging in with the same user will replace the existing
     node.
 - Remove support for Tailscale clients older than 1.62 (Capability version 87)
-      [#2405](https://github.com/juanfont/headscale/pull/2405)
+  [#2405](https://github.com/juanfont/headscale/pull/2405)
 
 ### Changes
 
@@ -49,6 +52,7 @@
 ## 0.24.3 (2025-02-07)
 
 ### Changes
+
 - Fix migration error caused by nodes having invalid auth keys
   [#2412](https://github.com/juanfont/headscale/pull/2412)
 - Pre auth keys belonging to a user are no longer deleted with the user

flake.nix

@@ -30,7 +30,7 @@
 
           # When updating go.mod or go.sum, a new sha will need to be calculated,
           # update this if you have a mismatch after doing a change to those files.
-          vendorHash = "sha256-ZQj2A0GdLhHc7JLW7qgpGBveXXNWg9ueSG47OZQQXEw=";
+          vendorHash = "sha256-CoxqEAxGdefyiIhz84LXXxPrZ1JWsX8Ernv1USr9JTs=";
 
           subPackages = ["cmd/headscale"];
 

go.mod

@@ -89,6 +89,7 @@ require (
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/akutz/memconn v0.1.0 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect
+	github.com/arl/statsviz v0.6.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
@@ -141,6 +142,7 @@ require (
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/gorilla/csrf v1.7.3-0.20250123201450-9dd6af1f6d30 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.2.0 // indirect
 	github.com/illarion/gonotify/v2 v2.0.3 // indirect

go.sum

@@ -41,6 +41,8 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE=
+github.com/arl/statsviz v0.6.0/go.mod h1:0toboo+YGSUXDaS4g1D5TVS4dXs7S7YYT5J/qnW2h8s=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
 github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
@@ -242,6 +244,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
 github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE=

hscontrol/app.go

@@ -36,7 +36,6 @@ import (
 	"github.com/juanfont/headscale/hscontrol/util"
 	zerolog "github.com/philip-bui/grpc-zerolog"
 	"github.com/pkg/profile"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	zl "github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/crypto/acme"
@@ -786,26 +785,12 @@ func (h *Headscale) Serve() error {
 	log.Info().
 		Msgf("listening and serving HTTP on: %s", h.cfg.Addr)
 
-	debugMux := http.NewServeMux()
-	debugMux.Handle("/debug/pprof/", http.DefaultServeMux)
-	debugMux.HandleFunc("/debug/notifier", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte(h.nodeNotifier.String()))
-	})
-	debugMux.Handle("/metrics", promhttp.Handler())
-
-	debugHTTPServer := &http.Server{
-		Addr:         h.cfg.MetricsAddr,
-		Handler:      debugMux,
-		ReadTimeout:  types.HTTPTimeout,
-		WriteTimeout: 0,
-	}
-
 	debugHTTPListener, err := net.Listen("tcp", h.cfg.MetricsAddr)
 	if err != nil {
 		return fmt.Errorf("failed to bind to TCP address: %w", err)
 	}
 
+	debugHTTPServer := h.debugHTTPServer()
 	errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) })
 
 	log.Info().

hscontrol/debug.go

@@ -0,0 +1,120 @@
+package hscontrol
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/arl/statsviz"
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tsweb"
+)
+
+func (h *Headscale) debugHTTPServer() *http.Server {
+	debugMux := http.NewServeMux()
+	debug := tsweb.Debugger(debugMux)
+	debug.Handle("notifier", "Connected nodes in notifier", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(h.nodeNotifier.String()))
+	}))
+	debug.Handle("config", "Current configuration", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		config, err := json.MarshalIndent(h.cfg, "", "  ")
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(config)
+	}))
+	debug.Handle("policy", "Current policy", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		pol, err := h.policyBytes()
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(pol)
+	}))
+	debug.Handle("filter", "Current filter", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		filter := h.polMan.Filter()
+
+		filterJSON, err := json.MarshalIndent(filter, "", "  ")
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(filterJSON)
+	}))
+	debug.Handle("ssh", "SSH Policy per node", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		nodes, err := h.db.ListNodes()
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+
+		sshPol := make(map[string]*tailcfg.SSHPolicy)
+		for _, node := range nodes {
+			pol, err := h.polMan.SSHPolicy(node)
+			if err != nil {
+				httpError(w, err)
+				return
+			}
+
+			sshPol[fmt.Sprintf("id:%d  hostname:%s givenname:%s", node.ID, node.Hostname, node.GivenName)] = pol
+		}
+
+		sshJSON, err := json.MarshalIndent(sshPol, "", "  ")
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(sshJSON)
+	}))
+	debug.Handle("derpmap", "Current DERPMap", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dm := h.DERPMap
+
+		dmJSON, err := json.MarshalIndent(dm, "", "  ")
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(dmJSON)
+	}))
+	debug.Handle("registration-cache", "Pending registrations", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		registrationsJSON, err := json.MarshalIndent(h.registrationCache.Items(), "", "  ")
+		if err != nil {
+			httpError(w, err)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write(registrationsJSON)
+	}))
+
+	err := statsviz.Register(debugMux)
+	if err == nil {
+		debug.URL("/debug/statsviz", "Statsviz (visualise go metrics)")
+	}
+
+	debug.URL("/metrics", "Prometheus metrics")
+	debugMux.Handle("/metrics", promhttp.Handler())
+
+	debugHTTPServer := &http.Server{
+		Addr:         h.cfg.MetricsAddr,
+		Handler:      debugMux,
+		ReadTimeout:  types.HTTPTimeout,
+		WriteTimeout: 0,
+	}
+
+	return debugHTTPServer
+}