Skip to content

Commit ff97cfb

Browse files
zqztenzqzten
committed
introduce metrics
Signed-off-by: Zach Zhu <[email protected]>
1 parent dc877c3 commit ff97cfb

File tree

16 files changed

+238
-23
lines changed

16 files changed

+238
-23
lines changed

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee
1313
github.com/nats-io/nats.go v1.13.1-0.20220318132711-e0e03e374228
1414
github.com/pkg/errors v0.9.1
15+
github.com/prometheus/client_golang v1.11.0
1516
github.com/rancher/wrangler v0.8.3
1617
github.com/shengdoushi/base58 v1.0.0
1718
github.com/sirupsen/logrus v1.7.0

‎main.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,20 @@ import (
55
"errors"
66
"fmt"
77
"os"
8+
"time"
89

10+
"github.com/k3s-io/kine/pkg/drivers/generic"
911
"github.com/k3s-io/kine/pkg/endpoint"
12+
"github.com/k3s-io/kine/pkg/metrics"
1013
"github.com/k3s-io/kine/pkg/version"
1114
"github.com/rancher/wrangler/pkg/signals"
1215
"github.com/sirupsen/logrus"
1316
"github.com/urfave/cli"
1417
)
1518

1619
var (
17-
config endpoint.Config
20+
config endpoint.Config
21+
metricsConfig metrics.Config
1822
)
1923

2024
func main() {
@@ -76,6 +80,18 @@ func main() {
7680
Usage: "Key file for DB connection",
7781
Destination: &config.BackendTLSConfig.KeyFile,
7882
},
83+
cli.StringFlag{
84+
Name: "metrics-bind-address",
85+
Usage: "The address the metric endpoint binds to. Default :8080, set 0 to disable metrics serving.",
86+
Destination: &metricsConfig.ServerAddress,
87+
Value: ":8080",
88+
},
89+
cli.DurationFlag{
90+
Name: "slow-sql-threshold",
91+
Usage: "The duration which SQL executed longer than will be logged. Default 1s, set <= 0 to disable slow SQL log.",
92+
Destination: &generic.SlowSQLThreshold,
93+
Value: time.Second,
94+
},
7995
cli.BoolFlag{Name: "debug"},
8096
}
8197
app.Action = run
@@ -92,6 +108,9 @@ func run(c *cli.Context) error {
92108
logrus.SetLevel(logrus.TraceLevel)
93109
}
94110
ctx := signals.SetupSignalHandler(context.Background())
111+
metricsConfig.ServerTLSConfig = config.ServerTLSConfig
112+
go metrics.Serve(ctx, metricsConfig)
113+
config.MetricsRegisterer = metrics.Registry
95114
_, err := endpoint.Listen(ctx, config)
96115
if err != nil {
97116
return err

‎pkg/drivers/dqlite/dqlite.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/k3s-io/kine/pkg/drivers/sqlite"
2020
"github.com/k3s-io/kine/pkg/server"
2121
"github.com/pkg/errors"
22+
"github.com/prometheus/client_golang/prometheus"
2223
"github.com/sirupsen/logrus"
2324
)
2425

@@ -68,7 +69,7 @@ outer:
6869
return nil
6970
}
7071

71-
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
72+
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
7273
opts, err := parseOpts(datasourceName)
7374
if err != nil {
7475
return nil, err
@@ -97,7 +98,7 @@ func New(ctx context.Context, datasourceName string, connPoolConfig generic.Conn
9798
}
9899

99100
sql.Register("dqlite", d)
100-
backend, generic, err := sqlite.NewVariant(ctx, "dqlite", opts.dsn, connPoolConfig)
101+
backend, generic, err := sqlite.NewVariant(ctx, "dqlite", opts.dsn, connPoolConfig, metricsRegisterer)
101102
if err != nil {
102103
return nil, errors.Wrap(err, "sqlite client")
103104
}

‎pkg/drivers/dqlite/no_dqlite.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ import (
99

1010
"github.com/k3s-io/kine/pkg/drivers/generic"
1111
"github.com/k3s-io/kine/pkg/server"
12+
"github.com/prometheus/client_golang/prometheus"
1213
)
1314

14-
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
15+
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
1516
return nil, errors.New(`this binary is built without dqlite support, compile with "-tags dqlite"`)
1617
}

‎pkg/drivers/generic/generic.go

+19-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
"github.com/Rican7/retry/backoff"
1515
"github.com/Rican7/retry/strategy"
1616
"github.com/k3s-io/kine/pkg/server"
17+
"github.com/prometheus/client_golang/prometheus"
18+
"github.com/prometheus/client_golang/prometheus/collectors"
1719
"github.com/sirupsen/logrus"
1820
)
1921

@@ -179,7 +181,7 @@ func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
179181
return db, nil
180182
}
181183

182-
func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool) (*Generic, error) {
184+
func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool, metricsRegisterer prometheus.Registerer) (*Generic, error) {
183185
var (
184186
db *sql.DB
185187
err error
@@ -201,6 +203,10 @@ func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig
201203

202204
configureConnectionPooling(connPoolConfig, db, driverName)
203205

206+
if metricsRegisterer != nil {
207+
metricsRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))
208+
}
209+
204210
return &Generic{
205211
DB: db,
206212

@@ -248,13 +254,21 @@ func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig
248254
}, err
249255
}
250256

251-
func (d *Generic) query(ctx context.Context, sql string, args ...interface{}) (*sql.Rows, error) {
257+
func (d *Generic) query(ctx context.Context, sql string, args ...interface{}) (result *sql.Rows, err error) {
252258
logrus.Tracef("QUERY %v : %s", args, Stripped(sql))
259+
startTime := time.Now()
260+
defer func() {
261+
ObserveSQL(startTime, err == nil, Stripped(sql), args)
262+
}()
253263
return d.DB.QueryContext(ctx, sql, args...)
254264
}
255265

256-
func (d *Generic) queryRow(ctx context.Context, sql string, args ...interface{}) *sql.Row {
266+
func (d *Generic) queryRow(ctx context.Context, sql string, args ...interface{}) (result *sql.Row) {
257267
logrus.Tracef("QUERY ROW %v : %s", args, Stripped(sql))
268+
startTime := time.Now()
269+
defer func() {
270+
ObserveSQL(startTime, result.Err() == nil, Stripped(sql), args)
271+
}()
258272
return d.DB.QueryRowContext(ctx, sql, args...)
259273
}
260274

@@ -267,7 +281,9 @@ func (d *Generic) execute(ctx context.Context, sql string, args ...interface{})
267281
wait := strategy.Backoff(backoff.Linear(100 + time.Millisecond))
268282
for i := uint(0); i < 20; i++ {
269283
logrus.Tracef("EXEC (try: %d) %v : %s", i, args, Stripped(sql))
284+
startTime := time.Now()
270285
result, err = d.DB.ExecContext(ctx, sql, args...)
286+
ObserveSQL(startTime, err == nil, Stripped(sql), args)
271287
if err != nil && d.Retry != nil && d.Retry(err) {
272288
wait(i)
273289
continue

‎pkg/drivers/generic/metrics.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package generic
2+
3+
import (
4+
"time"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
"github.com/sirupsen/logrus"
8+
)
9+
10+
const (
11+
LabelSuccess = "success"
12+
LabelError = "error"
13+
)
14+
15+
var (
16+
SQLTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
17+
Name: "kine_sql_total",
18+
Help: "Total number of SQL operations",
19+
}, []string{"result"})
20+
21+
SQLTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
22+
Name: "kine_sql_time_seconds",
23+
Help: "Length of time per SQL operation",
24+
Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
25+
1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30},
26+
}, []string{"result"})
27+
)
28+
29+
var (
30+
// SlowSQLThreshold is a duration which SQL executed longer than will be logged.
31+
// This can be directly modified to override the default value when kine is used as a library.
32+
SlowSQLThreshold = time.Second
33+
)
34+
35+
func ObserveSQL(start time.Time, success bool, sql Stripped, args ...interface{}) {
36+
result := LabelSuccess
37+
if !success {
38+
result = LabelError
39+
}
40+
SQLTotal.WithLabelValues(result).Inc()
41+
duration := time.Since(start)
42+
SQLTime.WithLabelValues(result).Observe(duration.Seconds())
43+
if SlowSQLThreshold > 0 && duration >= SlowSQLThreshold {
44+
logrus.Infof("Slow SQL (started: %v) (total time: %v): %s : %v", start, duration, sql, args)
45+
}
46+
}

‎pkg/drivers/generic/tx.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package generic
33
import (
44
"context"
55
"database/sql"
6+
"time"
67

78
"github.com/k3s-io/kine/pkg/server"
89
"github.com/sirupsen/logrus"
@@ -97,17 +98,29 @@ func (t *Tx) CurrentRevision(ctx context.Context) (int64, error) {
9798
return id, err
9899
}
99100

100-
func (t *Tx) query(ctx context.Context, sql string, args ...interface{}) (*sql.Rows, error) {
101+
func (t *Tx) query(ctx context.Context, sql string, args ...interface{}) (result *sql.Rows, err error) {
101102
logrus.Tracef("TX QUERY %v : %s", args, Stripped(sql))
103+
startTime := time.Now()
104+
defer func() {
105+
ObserveSQL(startTime, err == nil, Stripped(sql), args)
106+
}()
102107
return t.x.QueryContext(ctx, sql, args...)
103108
}
104109

105-
func (t *Tx) queryRow(ctx context.Context, sql string, args ...interface{}) *sql.Row {
110+
func (t *Tx) queryRow(ctx context.Context, sql string, args ...interface{}) (result *sql.Row) {
106111
logrus.Tracef("TX QUERY ROW %v : %s", args, Stripped(sql))
112+
startTime := time.Now()
113+
defer func() {
114+
ObserveSQL(startTime, result.Err() == nil, Stripped(sql), args)
115+
}()
107116
return t.x.QueryRowContext(ctx, sql, args...)
108117
}
109118

110119
func (t *Tx) execute(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error) {
111120
logrus.Tracef("TX EXEC %v : %s", args, Stripped(sql))
121+
startTime := time.Now()
122+
defer func() {
123+
ObserveSQL(startTime, err == nil, Stripped(sql), args)
124+
}()
112125
return t.x.ExecContext(ctx, sql, args...)
113126
}

‎pkg/drivers/mysql/mysql.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/k3s-io/kine/pkg/logstructured/sqllog"
1212
"github.com/k3s-io/kine/pkg/server"
1313
"github.com/k3s-io/kine/pkg/tls"
14+
"github.com/prometheus/client_golang/prometheus"
1415
"github.com/sirupsen/logrus"
1516
)
1617

@@ -43,7 +44,7 @@ var (
4344
createDB = "CREATE DATABASE IF NOT EXISTS "
4445
)
4546

46-
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
47+
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
4748
tlsConfig, err := tlsInfo.ClientConfig()
4849
if err != nil {
4950
return nil, err
@@ -62,7 +63,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
6263
return nil, err
6364
}
6465

65-
dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false)
66+
dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false, metricsRegisterer)
6667
if err != nil {
6768
return nil, err
6869
}

‎pkg/drivers/pgsql/pgsql.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/k3s-io/kine/pkg/server"
1515
"github.com/k3s-io/kine/pkg/tls"
1616
"github.com/lib/pq"
17+
"github.com/prometheus/client_golang/prometheus"
1718
"github.com/sirupsen/logrus"
1819
)
1920

@@ -44,7 +45,7 @@ var (
4445
createDB = "CREATE DATABASE "
4546
)
4647

47-
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
48+
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
4849
parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
4950
if err != nil {
5051
return nil, err
@@ -54,7 +55,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
5455
return nil, err
5556
}
5657

57-
dialect, err := generic.Open(ctx, "postgres", parsedDSN, connPoolConfig, "$", true)
58+
dialect, err := generic.Open(ctx, "postgres", parsedDSN, connPoolConfig, "$", true, metricsRegisterer)
5859
if err != nil {
5960
return nil, err
6061
}

‎pkg/drivers/sqlite/sqlite.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/k3s-io/kine/pkg/server"
1616
"github.com/mattn/go-sqlite3"
1717
"github.com/pkg/errors"
18+
"github.com/prometheus/client_golang/prometheus"
1819
"github.com/sirupsen/logrus"
1920

2021
// sqlite db driver
@@ -44,20 +45,20 @@ var (
4445
}
4546
)
4647

47-
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
48-
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig)
48+
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
49+
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig, metricsRegisterer)
4950
return backend, err
5051
}
5152

52-
func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
53+
func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, *generic.Generic, error) {
5354
if dataSourceName == "" {
5455
if err := os.MkdirAll("./db", 0700); err != nil {
5556
return nil, nil, err
5657
}
5758
dataSourceName = "./db/state.db?_journal=WAL&cache=shared"
5859
}
5960

60-
dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false)
61+
dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false, metricsRegisterer)
6162
if err != nil {
6263
return nil, nil, err
6364
}

‎pkg/drivers/sqlite/sqlite_nocgo.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,16 @@ import (
1010

1111
"github.com/k3s-io/kine/pkg/drivers/generic"
1212
"github.com/k3s-io/kine/pkg/server"
13+
"github.com/prometheus/client_golang/prometheus"
1314
)
1415

1516
var errNoCgo = errors.New("this binary is built without CGO, sqlite is disabled")
1617

17-
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
18+
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
1819
return nil, errNoCgo
1920
}
2021

21-
func NewVariant(driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
22+
func NewVariant(driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, *generic.Generic, error) {
2223
return nil, nil, errNoCgo
2324
}
2425

0 commit comments

Comments
 (0)