Skip to content

Commit 9e264a4

Browse files
lidellidel
committed
feat(acme): improved logging and http metrics
rerequisite for productization and monitoring this is necessary minimum to gain some understanding about how long requests take, how many are in flight /v1/health endpoint added for monitoring by load-balancer
1 parent 5604d3d commit 9e264a4

File tree

7 files changed

+71
-26
lines changed

7 files changed

+71
-26
lines changed

‎README.md

+24
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,25 @@
77
This is the backend of [`AutoTLS` feature introduced in Kubo 0.32.0-rc1](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls).
88
It is deployed at `libp2p.direct` and maintained by [Interplanetary Shipyard](https://github.com/ipshipyard).
99

10+
- [High-level Design](#high-level-design)
11+
- [Peer Authentication and DNS-01 Challenge and Certificate Issuance](#peer-authentication-and-dns-01-challenge-and-certificate-issuance)
12+
- [DNS Resolution and TLS Connection](#dns-resolution-and-tls-connection)
13+
- [Build](#build)
14+
- [Install](#install)
15+
- [From source](#from-source)
16+
- [Usage](#usage)
17+
- [Local testing](#local-testing)
18+
- [Docker](#docker)
19+
- [Configuration](#configuration)
20+
- [ipparser Syntax](#ipparser-syntax)
21+
- [acme Syntax](#acme-syntax)
22+
- [Example](#example)
23+
- [Handled DNS records](#handled-dns-records)
24+
- [IPv4 subdomain handling](#ipv4-subdomain-handling)
25+
- [IPv6 subdomain handling](#ipv6-subdomain-handling)
26+
- [Submitting Challenge Records](#submitting-challenge-records)
27+
- [Health Check](#health-check)
28+
1029
## High-level Design
1130

1231
The following diagrams show the high-level design of how p2p-forge works.
@@ -211,3 +230,8 @@ curl -X POST "https://registration.libp2p.direct/v1/_acme-challenge" \
211230
```
212231

213232
Where the bearer token is derived via the [libp2p HTTP PeerID Auth Specification](https://github.com/libp2p/specs/blob/master/http/peer-id-auth.md).
233+
234+
### Health Check
235+
236+
`/v1/health` will always respond with HTTP 204
237+

‎acme/metrics.go

-6
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,6 @@ import (
1010
)
1111

1212
var (
13-
registrationRequestCount = promauto.NewCounterVec(prometheus.CounterOpts{
14-
Namespace: plugin.Namespace,
15-
Subsystem: "forge_" + pluginName,
16-
Name: "registrations_total",
17-
Help: "Counter of ACME DNS-01 broker registration requests made to this p2p-forge instance.",
18-
}, []string{"status"})
1913
dns01ResponseCount = promauto.NewCounterVec(prometheus.CounterOpts{
2014
Namespace: plugin.Namespace,
2115
Subsystem: "forge_" + pluginName,

‎acme/writer.go

+29-11
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,18 @@ import (
1212
"net"
1313
"net/http"
1414
"os"
15-
"strconv"
1615
"strings"
1716
"time"
1817

1918
clog "github.com/coredns/coredns/plugin/pkg/log"
2019
"github.com/coredns/coredns/plugin/pkg/reuseport"
2120
"github.com/felixge/httpsnoop"
2221
"github.com/ipshipyard/p2p-forge/client"
22+
"github.com/prometheus/client_golang/prometheus"
23+
24+
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
25+
"github.com/slok/go-http-metrics/middleware"
26+
"github.com/slok/go-http-metrics/middleware/std"
2327

2428
"github.com/caddyserver/certmagic"
2529

@@ -34,6 +38,7 @@ import (
3438
var log = clog.NewWithPlugin(pluginName)
3539

3640
const registrationApiPath = "/v1/_acme-challenge"
41+
const healthcheckApiPath = "/v1/health"
3742

3843
// acmeWriter implements writing of ACME Challenge DNS records by exporting an HTTP endpoint.
3944
type acmeWriter struct {
@@ -88,8 +93,6 @@ func (c *acmeWriter) OnStartup() error {
8893
}
8994

9095
c.ln = ln
91-
92-
mux := http.NewServeMux()
9396
c.nlSetup = true
9497

9598
// server side secret key and peerID not particularly relevant, so we can generate new ones as needed
@@ -171,11 +174,24 @@ func (c *acmeWriter) OnStartup() error {
171174
}
172175
}
173176

177+
// middleware with prometheus recorder
178+
httpMetricsMiddleware := middleware.New(middleware.Config{
179+
Recorder: metrics.NewRecorder(metrics.Config{
180+
Registry: prometheus.DefaultRegisterer.(*prometheus.Registry),
181+
Prefix: "coredns_forge_" + pluginName,
182+
DurationBuckets: []float64{0.1, 0.5, 1, 2, 5, 8, 10, 20, 30}, // TODO: remove this comment if we are ok with these buckets
183+
}),
184+
DisableMeasureSize: true, // not meaningful for the registration api
185+
})
186+
174187
// register handlers
175-
mux.Handle(registrationApiPath, authPeer)
188+
mux := http.NewServeMux()
189+
mux.Handle(registrationApiPath, std.Handler(registrationApiPath, httpMetricsMiddleware, authPeer))
190+
mux.HandleFunc(healthcheckApiPath, func(w http.ResponseWriter, r *http.Request) {
191+
w.WriteHeader(http.StatusNoContent)
192+
})
176193

177-
// wrap handler in metrics meter
178-
c.handler = withRequestMetrics(mux)
194+
c.handler = withRequestLogger(mux)
179195

180196
go func() {
181197
log.Infof("Registration HTTP API (%s) listener at %s", registrationApiPath, c.ln.Addr().String())
@@ -185,12 +201,14 @@ func (c *acmeWriter) OnStartup() error {
185201
return nil
186202
}
187203

188-
func withRequestMetrics(next http.Handler) http.Handler {
204+
func withRequestLogger(next http.Handler) http.Handler {
189205
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
190-
m := httpsnoop.CaptureMetrics(next, w, r)
191-
registrationRequestCount.WithLabelValues(strconv.Itoa(m.Code)).Add(1)
192-
// TODO: decide if we keep below logger
193-
log.Infof("%s %s (status=%d dt=%s ua=%q)", r.Method, r.URL, m.Code, m.Duration, r.UserAgent())
206+
// we skip healthcheck endpoint because its spammed
207+
if !strings.HasPrefix(r.URL.Path, healthcheckApiPath) {
208+
// TODO: decide if we want to keep this logger enabled by default, or move it to debug
209+
m := httpsnoop.CaptureMetrics(next, w, r)
210+
log.Infof("%s %s (status=%d dt=%s ua=%q)", r.Method, r.URL, m.Code, m.Duration, r.UserAgent())
211+
}
194212
})
195213
}
196214

‎docs/METRICS.md

+7-1
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,15 @@ It includes default [Prometheus Glient metrics](https://prometheus.io/docs/guide
66

77
### Forge metrics
88

9+
#### `ipparser` plugin (DNS A/AAAA)
10+
911
- `coredns_forge_ipparser_requests_total{type}` - dynamic DNS `A`/`AAAA` responses generated by `ipparser` plugin for `ip.peerid.domain` and `peerid.domain`, including `NODATA-*` ones.
12+
13+
#### `acme` plugin (HTTP/ACME registration)
14+
15+
- `coredns_forge_acme_http_request_duration_seconds{code, handler}` - Histogram with the status, count and latency of the HTTP requests sent to path specified in `handler` (like `/v1/_acme-challenge`).
16+
- `coredns_forge_acme_http_requests_inflight{handler}` - The number of inflight requests being handled by `handler`.
1017
- `coredns_forge_acme_dns01_responses_total{type}` - DNS `TXT` responses generated by `acme` plugin for DNS-01 challenge at `_acme-challenge.peerid.domain`, including `NODATA-*` ones.
11-
- `coredns_forge_acme_registrations_total{status}` - registration API responses generated by `acme` plugin when a client attempts to register DNS-01 challenge for their PeerID at `_acme-challenge.peerid.domain`.
1218
- `coredns_forge_acme_libp2p_probe_total{result, agent}` - libp2p probe results `acme` plugin when testing connectivity before accepting DNS-01 challenge for a PeerID. `status` is either `ok` or `error` and `agent` value is limited to well known agents, `other` and `unknown`.
1319

1420
### CoreDNS metrics

‎e2e_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ func TestIPv4Lookup(t *testing.T) {
371371
return
372372
}
373373

374-
expectedAnswer := fmt.Sprintf(`%s 3600 IN A %s`, m.Question[0].Name, tt.expectedAddress)
374+
expectedAnswer := fmt.Sprintf(`%s 86400 IN A %s`, m.Question[0].Name, tt.expectedAddress)
375375
if r.Answer[0].String() != expectedAnswer {
376376
t.Fatalf("Expected %s reply, got %s", expectedAnswer, r.Answer[0].String())
377377
}
@@ -487,7 +487,7 @@ func TestIPv6Lookup(t *testing.T) {
487487
return
488488
}
489489

490-
expectedAnswer := fmt.Sprintf(`%s 3600 IN AAAA %s`, m.Question[0].Name, tt.expectedAddress)
490+
expectedAnswer := fmt.Sprintf(`%s 86400 IN AAAA %s`, m.Question[0].Name, tt.expectedAddress)
491491
if r.Answer[0].String() != expectedAnswer {
492492
t.Fatalf("Expected %s reply, got %s", expectedAnswer, r.Answer[0].String())
493493
}

‎go.mod

+3-2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ require (
2020
github.com/multiformats/go-multiaddr-dns v0.4.0
2121
github.com/multiformats/go-multibase v0.2.0
2222
github.com/prometheus/client_golang v1.20.5
23+
github.com/slok/go-http-metrics v0.13.0
2324
go.uber.org/zap v1.27.0
2425
)
2526

@@ -63,7 +64,7 @@ require (
6364
github.com/dustin/go-humanize v1.0.1 // indirect
6465
github.com/ebitengine/purego v0.6.0-alpha.5 // indirect
6566
github.com/elastic/gosigar v0.14.3 // indirect
66-
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
67+
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
6768
github.com/farsightsec/golang-framestream v0.3.0 // indirect
6869
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
6970
github.com/flynn/noise v1.1.0 // indirect
@@ -203,7 +204,7 @@ require (
203204
golang.org/x/sys v0.26.0 // indirect
204205
golang.org/x/term v0.25.0 // indirect
205206
golang.org/x/text v0.19.0 // indirect
206-
golang.org/x/time v0.5.0 // indirect
207+
golang.org/x/time v0.6.0 // indirect
207208
golang.org/x/tools v0.26.0 // indirect
208209
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
209210
google.golang.org/api v0.172.0 // indirect

‎go.sum

+6-4
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUk
128128
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
129129
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
130130
github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
131-
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
132-
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
131+
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
132+
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
133133
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
134134
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
135135
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -531,6 +531,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
531531
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
532532
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
533533
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
534+
github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8=
535+
github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4=
534536
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
535537
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
536538
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
@@ -751,8 +753,8 @@ golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
751753
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
752754
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
753755
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
754-
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
755-
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
756+
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
757+
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
756758
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
757759
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
758760
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

0 commit comments

Comments
 (0)