Skip to content

Commit e9a9d21

Browse files
Swopxvzf
Swop
authored and
xvzf
committed
feat: prometheus metrics, route names in logger & panic bug fix (#5)
* feat: adds prometheus metrics * chore(logger): move logger middleware in middlewares package + add route names to be compliant with company's aks_route_name field * fix: prevent panic errors happening in the server init phase to be swallowed
1 parent 3ca6190 commit e9a9d21

File tree

9 files changed

+763
-17
lines changed

9 files changed

+763
-17
lines changed

‎go.mod

+10-3
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@ require (
66
github.com/dgraph-io/badger/v3 v3.2103.5
77
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46
88
github.com/go-playground/validator/v10 v10.11.2
9+
github.com/gofiber/adaptor/v2 v2.1.32
910
github.com/gofiber/fiber/v2 v2.42.0
1011
github.com/google/go-cmp v0.5.9
1112
github.com/google/go-github/v50 v50.0.0
1213
github.com/onsi/ginkgo/v2 v2.8.2
1314
github.com/onsi/gomega v1.27.1
15+
github.com/prometheus/client_golang v1.14.0
1416
github.com/rs/zerolog v1.29.0
1517
github.com/stretchr/testify v1.8.1
1618
golang.org/x/oauth2 v0.5.0
@@ -20,8 +22,9 @@ require (
2022

2123
require (
2224
github.com/andybalholm/brotli v1.0.4 // indirect
25+
github.com/beorn7/perks v1.0.1 // indirect
2326
github.com/cespare/xxhash v1.1.0 // indirect
24-
github.com/cespare/xxhash/v2 v2.1.1 // indirect
27+
github.com/cespare/xxhash/v2 v2.1.2 // indirect
2528
github.com/davecgh/go-spew v1.1.1 // indirect
2629
github.com/dgraph-io/ristretto v0.1.1 // indirect
2730
github.com/dustin/go-humanize v1.0.0 // indirect
@@ -31,7 +34,7 @@ require (
3134
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
3235
github.com/gogo/protobuf v1.3.2 // indirect
3336
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
34-
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
37+
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
3538
github.com/golang/protobuf v1.5.2 // indirect
3639
github.com/golang/snappy v0.0.3 // indirect
3740
github.com/google/flatbuffers v1.12.1 // indirect
@@ -44,9 +47,13 @@ require (
4447
github.com/mattn/go-colorable v0.1.13 // indirect
4548
github.com/mattn/go-isatty v0.0.17 // indirect
4649
github.com/mattn/go-runewidth v0.0.14 // indirect
50+
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
4751
github.com/philhofer/fwd v1.1.1 // indirect
4852
github.com/pkg/errors v0.9.1 // indirect
4953
github.com/pmezard/go-difflib v1.0.0 // indirect
54+
github.com/prometheus/client_model v0.3.0 // indirect
55+
github.com/prometheus/common v0.37.0 // indirect
56+
github.com/prometheus/procfs v0.8.0 // indirect
5057
github.com/rivo/uniseg v0.2.0 // indirect
5158
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
5259
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d // indirect
@@ -61,5 +68,5 @@ require (
6168
golang.org/x/text v0.7.0 // indirect
6269
golang.org/x/tools v0.6.0 // indirect
6370
google.golang.org/appengine v1.6.7 // indirect
64-
google.golang.org/protobuf v1.28.0 // indirect
71+
google.golang.org/protobuf v1.28.1 // indirect
6572
)

‎go.sum

+387-4
Large diffs are not rendered by default.

‎internal/lease/leaseprovider.go

+35-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type ProviderOpts struct {
2222
ID string
2323
Clock clock.PassiveClock
2424
Storage storage.Storage[*ProviderState]
25+
Metrics *providerMetrics
2526
}
2627

2728
type Status string
@@ -161,6 +162,7 @@ type leaseProviderImpl struct {
161162
opts ProviderOpts
162163
clock clock.PassiveClock
163164
storage storage.Storage[*ProviderState]
165+
metrics *providerMetrics
164166

165167
state *ProviderState
166168
}
@@ -181,6 +183,7 @@ func NewLeaseProvider(opts ProviderOpts) Provider {
181183
opts: opts,
182184
clock: cl,
183185
storage: st,
186+
metrics: opts.Metrics,
184187
state: &ProviderState{
185188
id: opts.ID,
186189
lastUpdatedAt: cl.Now(),
@@ -190,7 +193,11 @@ func NewLeaseProvider(opts ProviderOpts) Provider {
190193
}
191194

192195
func (lp *leaseProviderImpl) HydrateFromState(ctx context.Context) error {
193-
return lp.storage.Hydrate(ctx, lp.state)
196+
if err := lp.storage.Hydrate(ctx, lp.state); err != nil {
197+
return err
198+
}
199+
lp.updateMetrics()
200+
return nil
194201
}
195202

196203
// MarshalJSON used to marshall the provider to its JSON form (used in API responses)
@@ -401,10 +408,24 @@ func (lp *leaseProviderImpl) evaluateRequest(ctx context.Context, req *Request)
401408
return req
402409
}
403410

411+
func (lp *leaseProviderImpl) updateMetrics() {
412+
if lp.metrics != nil {
413+
queueSize := 0
414+
for _, r := range lp.state.known {
415+
if pointer.StringDeref(r.Status, StatusCompleted) != StatusCompleted {
416+
queueSize++
417+
}
418+
}
419+
420+
lp.metrics.queueSize.WithLabelValues(lp.opts.ID).Set(float64(queueSize))
421+
}
422+
}
423+
404424
func (lp *leaseProviderImpl) Acquire(ctx context.Context, leaseRequest *Request) (*Request, error) {
405425
// Ensure we don't have any collisions
406426
lp.mutex.Lock()
407427
defer lp.mutex.Unlock()
428+
defer lp.updateMetrics()
408429

409430
// Save the state to storage
410431
defer lp.saveState(ctx)
@@ -431,6 +452,7 @@ func (lp *leaseProviderImpl) Acquire(ctx context.Context, leaseRequest *Request)
431452
func (lp *leaseProviderImpl) Release(ctx context.Context, leaseRequest *Request) (*Request, error) {
432453
lp.mutex.Lock()
433454
defer lp.mutex.Unlock()
455+
defer lp.updateMetrics()
434456

435457
// Save the state to storage
436458
defer lp.saveState(ctx)
@@ -455,6 +477,18 @@ func (lp *leaseProviderImpl) Release(ctx context.Context, leaseRequest *Request)
455477
if status == StatusSuccess {
456478
// On success, set status to completed so all remaining ones can be removed
457479
req.Status = pointer.String(StatusCompleted)
480+
481+
if lp.metrics != nil {
482+
// compute merged batch size to report in the metrics
483+
mergedBatchSize := 1
484+
for _, known := range lp.state.known {
485+
if known.Priority < req.Priority {
486+
mergedBatchSize++
487+
}
488+
}
489+
lp.metrics.mergedBatchSize.WithLabelValues(lp.opts.ID).Observe(float64(mergedBatchSize))
490+
}
491+
458492
return req, nil
459493
}
460494

‎internal/lease/leaseproviderorchestrator.go

+30
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,46 @@ import (
77
"time"
88

99
"github.com/ankorstore/gh-action-mq-lease-service/internal/config/server/latest"
10+
"github.com/ankorstore/gh-action-mq-lease-service/internal/metrics"
1011
"github.com/ankorstore/gh-action-mq-lease-service/internal/storage"
12+
"github.com/prometheus/client_golang/prometheus"
1113
"k8s.io/utils/clock"
1214
)
1315

1416
type NewProviderOrchestratorOpts struct {
1517
Repositories []*latest.GithubRepositoryConfig
1618
Clock clock.PassiveClock
1719
Storage storage.Storage[*ProviderState]
20+
Metrics metrics.Metrics
21+
}
22+
23+
type providerMetrics struct {
24+
queueSize *prometheus.GaugeVec
25+
mergedBatchSize *prometheus.HistogramVec
1826
}
1927

2028
func NewProviderOrchestrator(opts NewProviderOrchestratorOpts) ProviderOrchestrator {
29+
var pMetrics *providerMetrics
30+
if opts.Metrics != nil {
31+
pMetrics = &providerMetrics{
32+
queueSize: opts.Metrics.NewGaugeVec(
33+
prometheus.GaugeOpts{
34+
Name: "provider_lease_requests_total",
35+
Help: "All lease requests known in a provider",
36+
},
37+
[]string{"provider_id"},
38+
),
39+
mergedBatchSize: opts.Metrics.NewHistogramVec(
40+
prometheus.HistogramOpts{
41+
Name: "provider_merged_batch_size",
42+
Help: "Number of requests merged in same batch",
43+
Buckets: []float64{1, 2, 3, 4, 5, 6, 7, 10, 15, 20},
44+
},
45+
[]string{"provider_id"},
46+
),
47+
}
48+
}
49+
2150
leaseProviders := make(map[string]Provider)
2251
for _, repository := range opts.Repositories {
2352
key := getKey(repository.Owner, repository.Name, repository.BaseRef)
@@ -28,6 +57,7 @@ func NewProviderOrchestrator(opts NewProviderOrchestratorOpts) ProviderOrchestra
2857
ID: key,
2958
Clock: opts.Clock,
3059
Storage: opts.Storage,
60+
Metrics: pMetrics,
3161
})
3262
}
3363
return &leaseProviderOrchestratorImpl{

0 commit comments

Comments
 (0)