From 817a17f1a17a24fbd37ad37afa89c8b98c3861df Mon Sep 17 00:00:00 2001 From: taddari Date: Thu, 27 Aug 2020 17:21:20 +0100 Subject: [PATCH 1/6] Add opencensus integration --- go.mod | 4 +++- go.sum | 13 +++++++++++++ pkg/database/database.go | 38 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8f3f9930d..7c6d375a1 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go v0.63.0 cloud.google.com/go/firestore v1.3.0 // indirect contrib.go.opencensus.io/exporter/stackdriver v0.13.3 // indirect + contrib.go.opencensus.io/integrations/ocsql v0.1.6 firebase.google.com/go v3.13.0+incompatible github.com/Azure/azure-sdk-for-go v45.1.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.4 // indirect @@ -31,7 +32,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.14.7 // indirect github.com/hashicorp/go-retryablehttp v0.6.7 // indirect github.com/jinzhu/gorm v1.9.16 - github.com/lib/pq v1.8.0 // indirect + github.com/lib/pq v1.8.0 github.com/lstoll/awskms v0.0.0-20200603175638-a388516467f1 // indirect github.com/mattn/go-colorable v0.1.7 // indirect github.com/mikehelmick/go-chaff v0.3.0 @@ -42,6 +43,7 @@ require ( github.com/ory/dockertest v3.3.5+incompatible github.com/prometheus/common v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.17.0 // indirect + github.com/sagikazarmark/go-gin-gorm-opencensus v0.0.0-20190530122219-5cd3c9b414a5 github.com/sethvargo/go-envconfig v0.3.1 github.com/sethvargo/go-limiter v0.4.1 github.com/sethvargo/go-redisstore v0.1.2-opencensus diff --git a/go.sum b/go.sum index 6aa56fb52..f57df96ff 100644 --- a/go.sum +++ b/go.sum @@ -46,14 +46,18 @@ cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f h1:UrKzEwTgeiff9vxdrfdqxibzpWjxLnuXDI5m6z3GJAk= code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI= +contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= contrib.go.opencensus.io/exporter/prometheus v0.2.0 h1:9PUk0/8V0LGoPqVCrf8fQZJkFGBxudu8jOjQSMwoD6w= contrib.go.opencensus.io/exporter/prometheus v0.2.0/go.mod h1:TYmVAyE8Tn1lyPcltF5IYYfWp2KHu7lQGIZnj8iZMys= contrib.go.opencensus.io/exporter/stackdriver v0.13.1 h1:RX9W6FelAqTVnBi/bRXJLXr9n18v4QkQwZYIdnNS51I= contrib.go.opencensus.io/exporter/stackdriver v0.13.1/go.mod h1:z2tyTZtPmQ2HvWH4cOmVDgtY+1lomfKdbLnkJvZdc8c= contrib.go.opencensus.io/exporter/stackdriver v0.13.3 h1:WDNe3wMJ0i8WQt9WmdOWoFzyqllcL+W9xzF92GzsU10= contrib.go.opencensus.io/exporter/stackdriver v0.13.3/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= +contrib.go.opencensus.io/integrations/ocsql v0.1.6 h1:9qmZJBlnMtffShflmfhW4EZK7M+CujIDG4bEwUrg+ms= +contrib.go.opencensus.io/integrations/ocsql v0.1.6/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= @@ -369,8 +373,10 @@ github.com/gammazero/workerpool v0.0.0-20190406235159-88d534f22b56 h1:VzbudKn/nv github.com/gammazero/workerpool v0.0.0-20190406235159-88d534f22b56/go.mod h1:w9RqFVO2BM3xwWEcAB8Fwp0OviTBBEiRmSBDfbXnd3w= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck= @@ -851,6 +857,7 @@ github.com/jefferai/jsonx v1.0.1 h1:GvWkLWihoLqDG0BSP45TUQJH9qsINX50PVrFULgpc/I= github.com/jefferai/jsonx v1.0.1/go.mod h1:yFo3l2fcm7cZVHGq3HKLXE+Pd4RWuRjNBDHksM7XekQ= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= +github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= @@ -859,6 +866,7 @@ github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLl github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= +github.com/jinzhu/now v1.0.0/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1195,6 +1203,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sagikazarmark/go-gin-gorm-opencensus v0.0.0-20190530122219-5cd3c9b414a5 h1:Q8GI+Mq3SJkydD+BB7gdl7GMx87T5SJostkI19QNIVU= +github.com/sagikazarmark/go-gin-gorm-opencensus v0.0.0-20190530122219-5cd3c9b414a5/go.mod h1:qol1Vwu2ebBpDuumO8FFqiaYlHzvmRAvSWrNqYXnYlw= github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= @@ -1299,6 +1309,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -1795,6 +1806,8 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/gormigrate.v1 v1.6.0 h1:XpYM6RHQPmzwY7Uyu+t+xxMXc86JYFJn4nEc9HzQjsI= gopkg.in/gormigrate.v1 v1.6.0/go.mod h1:Lf00lQrHqfSYWiTtPcyQabsDdM6ejZaMgV0OU6JMSlw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= diff --git a/pkg/database/database.go b/pkg/database/database.go index dfeb424d1..c2d02bdbd 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -17,9 +17,11 @@ package database import ( "context" + "database/sql" "encoding/base64" "errors" "fmt" + "time" "github.com/google/exposure-notifications-server/pkg/base64util" "github.com/google/exposure-notifications-server/pkg/keys" @@ -30,7 +32,9 @@ import ( "go.uber.org/zap" // ensure the postgres dialiect is compiled in. - _ "github.com/jinzhu/gorm/dialects/postgres" + "contrib.go.opencensus.io/integrations/ocsql" + postgres "github.com/lib/pq" + "github.com/sagikazarmark/go-gin-gorm-opencensus/pkg/ocgorm" ) // Database is a handle to the database layer for the Exposure Notifications @@ -51,6 +55,8 @@ type Database struct { // secretManager is used to resolve secrets. secretManager secrets.SecretManager + + statsCloser func() } // SupportsPerRealmSigning returns true if the configuration supports @@ -120,12 +126,37 @@ func (db *Database) Open(ctx context.Context) error { return db.OpenWithCacher(ctx, nil) } +const driverName = "ocsql" + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + // OpenWithCacher creates a database connection with the cacher. This should // only be called once. func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) error { c := db.config - rawDB, err := gorm.Open("postgres", c.ConnectionString()) + driver := ocsql.Wrap(&postgres.Driver{}, ocsql.WithAllTraceOptions()) + if !stringInSlice(driverName, sql.Drivers()) { + ocsql.RegisterAllViews() + sql.Register(driverName, driver) + } + dbSql, err := sql.Open(driverName, c.ConnectionString()) + if err != nil { + return fmt.Errorf("Failed to open the SQL database: %v", err) + } + // enable periodic recording of sql.DBStats + db.statsCloser = ocsql.RecordStats(dbSql, 5*time.Second) + + //Need to give postgres dialect as otherwise gorm starts running + //in compatibility mode + rawDB, err := gorm.Open("postgres", dbSql) if err != nil { return fmt.Errorf("database gorm.Open: %w", err) } @@ -142,6 +173,8 @@ func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) err // Enable auto-preloading. rawDB = rawDB.Set("gorm:auto_preload", true) + ocgorm.RegisterCallbacks(rawDB) + callbacks := rawDB.Callback() // SMS configs @@ -183,6 +216,7 @@ func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) err // Close will close the database connection. Should be deferred right after Open. func (db *Database) Close() error { + db.statsCloser() return db.db.Close() } From 9111039447f67156ecea262f0eb015f3664bfa08 Mon Sep 17 00:00:00 2001 From: taddari Date: Thu, 27 Aug 2020 22:43:47 +0100 Subject: [PATCH 2/6] Removed trace options --- pkg/database/database.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index c2d02bdbd..95ad81a25 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -142,21 +142,21 @@ func stringInSlice(a string, list []string) bool { func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) error { c := db.config - driver := ocsql.Wrap(&postgres.Driver{}, ocsql.WithAllTraceOptions()) + driver := ocsql.Wrap(&postgres.Driver{}) if !stringInSlice(driverName, sql.Drivers()) { ocsql.RegisterAllViews() sql.Register(driverName, driver) } - dbSql, err := sql.Open(driverName, c.ConnectionString()) + dbSQL, err := sql.Open(driverName, c.ConnectionString()) if err != nil { - return fmt.Errorf("Failed to open the SQL database: %v", err) + return fmt.Errorf("failed to open the SQL database: %v", err) } // enable periodic recording of sql.DBStats - db.statsCloser = ocsql.RecordStats(dbSql, 5*time.Second) + db.statsCloser = ocsql.RecordStats(dbSQL, 5*time.Second) //Need to give postgres dialect as otherwise gorm starts running //in compatibility mode - rawDB, err := gorm.Open("postgres", dbSql) + rawDB, err := gorm.Open("postgres", dbSQL) if err != nil { return fmt.Errorf("database gorm.Open: %w", err) } From c1d6b52c3f03d5c330c96f3ca3ae599506e3a511 Mon Sep 17 00:00:00 2001 From: taddari Date: Mon, 7 Sep 2020 11:33:59 +0100 Subject: [PATCH 3/6] increase delay for test case --- pkg/database/token_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/database/token_test.go b/pkg/database/token_test.go index 4ff7bfb57..897f292fa 100644 --- a/pkg/database/token_test.go +++ b/pkg/database/token_test.go @@ -148,11 +148,11 @@ func TestIssueToken(t *testing.T) { LongCode: "00000002ABC", Claimed: false, TestType: "confirmed", - ExpiresAt: time.Now().Add(2 * time.Second), - LongExpiresAt: time.Now().Add(2 * time.Second), + ExpiresAt: time.Now().Add(5 * time.Second), + LongExpiresAt: time.Now().Add(5 * time.Second), }, Accept: acceptConfirmed, - Delay: 2 * time.Second, + Delay: 5 * time.Second, Error: ErrVerificationCodeExpired.Error(), TokenAge: time.Hour, }, From c09650285bbe4ed823df4eab6e8659e7e9299bf4 Mon Sep 17 00:00:00 2001 From: taddari Date: Tue, 8 Sep 2020 09:34:40 +0100 Subject: [PATCH 4/6] Ran go mod tidy --- go.sum | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/go.sum b/go.sum index 45db3b022..3db6a16da 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxo cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.33.1/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= @@ -50,12 +49,10 @@ cloud.google.com/go/storage v1.11.0 h1:bSLyzhbGjLMYxCratCDRSSH7+xRGpNApTBmowDUFG cloud.google.com/go/storage v1.11.0/go.mod h1:/PAbprKS+5msVYogBmczjWalDXnQ9mr64yEq9YnyPeo= code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f h1:UrKzEwTgeiff9vxdrfdqxibzpWjxLnuXDI5m6z3GJAk= code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI= -contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200615190824-f8c219d2d895 h1:bXMXgHq+WAIkI2dQivk1yMyKVVqMj8emLmhEqQghPWw= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200615190824-f8c219d2d895/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= -contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= contrib.go.opencensus.io/exporter/prometheus v0.2.0 h1:9PUk0/8V0LGoPqVCrf8fQZJkFGBxudu8jOjQSMwoD6w= contrib.go.opencensus.io/exporter/prometheus v0.2.0/go.mod h1:TYmVAyE8Tn1lyPcltF5IYYfWp2KHu7lQGIZnj8iZMys= contrib.go.opencensus.io/exporter/prometheus v0.2.1-0.20200609204449-6bcf6f8577f0 h1:2O3c1g5CzMc1+Uah4Waot9Obm0yw70VXJzWaP6Fz3nw= @@ -310,7 +307,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec h1:NfhRXXFDPxcF5Cwo06DzeIaE7uuJtAUhsDwH3LNsjos= @@ -382,9 +378,7 @@ github.com/gammazero/workerpool v0.0.0-20190406235159-88d534f22b56/go.mod h1:w9R github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= -github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= @@ -806,7 +800,6 @@ github.com/jefferai/jsonx v1.0.1 h1:GvWkLWihoLqDG0BSP45TUQJH9qsINX50PVrFULgpc/I= github.com/jefferai/jsonx v1.0.1/go.mod h1:yFo3l2fcm7cZVHGq3HKLXE+Pd4RWuRjNBDHksM7XekQ= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= -github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= @@ -814,7 +807,6 @@ github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLl github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= -github.com/jinzhu/now v1.0.0/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -1132,8 +1124,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/sagikazarmark/go-gin-gorm-opencensus v0.0.0-20190530122219-5cd3c9b414a5 h1:Q8GI+Mq3SJkydD+BB7gdl7GMx87T5SJostkI19QNIVU= -github.com/sagikazarmark/go-gin-gorm-opencensus v0.0.0-20190530122219-5cd3c9b414a5/go.mod h1:qol1Vwu2ebBpDuumO8FFqiaYlHzvmRAvSWrNqYXnYlw= github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= @@ -1224,7 +1214,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -1707,8 +1696,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/gormigrate.v1 v1.6.0 h1:XpYM6RHQPmzwY7Uyu+t+xxMXc86JYFJn4nEc9HzQjsI= gopkg.in/gormigrate.v1 v1.6.0/go.mod h1:Lf00lQrHqfSYWiTtPcyQabsDdM6ejZaMgV0OU6JMSlw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= From 08e9983d2818ea8fa1929676da9bb532b7206a67 Mon Sep 17 00:00:00 2001 From: taddari Date: Tue, 8 Sep 2020 17:07:19 +0100 Subject: [PATCH 5/6] Moved driver registration to init method and added retriable sql connection. --- pkg/database/database.go | 56 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 3fd900872..6f50ed218 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -59,6 +59,18 @@ type Database struct { statsCloser func() } +// Overrides the postgresql driver with +func init() { + const driverName = "ocsql" + for _, v := range sql.Drivers() { + if v == driverName { + return + } + } + ocsql.RegisterAllViews() + sql.Register(driverName, ocsql.Wrap(&postgres.Driver{})) +} + // SupportsPerRealmSigning returns true if the configuration supports // application managed signing keys. func (db *Database) SupportsPerRealmSigning() bool { @@ -126,17 +138,6 @@ func (db *Database) Open(ctx context.Context) error { return db.OpenWithCacher(ctx, nil) } -const driverName = "ocsql" - -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - // OpenWithCacher creates a database connection with the cacher. This should // only be called once. func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) error { @@ -144,33 +145,36 @@ func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) err // Establish a connection to the database. We use this later to register // opencenusus stats. - var rawDB *gorm.DB + var dbSQL *sql.DB if err := withRetries(ctx, func(ctx context.Context) error { var err error - driver := ocsql.Wrap(&postgres.Driver{}) - if !stringInSlice(driverName, sql.Drivers()) { - ocsql.RegisterAllViews() - sql.Register(driverName, driver) - } - dbSQL, err := sql.Open(driverName, c.ConnectionString()) + dbSQL, err = sql.Open("ocsql", c.ConnectionString()) if err != nil { - return fmt.Errorf("failed to open the SQL database: %v", err) - } - // enable periodic recording of sql.DBStats + return retry.RetryableError(err) + } // enable periodic recording of sql.DBStats db.statsCloser = ocsql.RecordStats(dbSQL, 5*time.Second) - - //Need to give postgres dialect as otherwise gorm starts running - //in compatibility mode + return nil + }); err != nil { + return fmt.Errorf("failed to open connection to database: %w", err) + } + if dbSQL == nil { + return fmt.Errorf("failed to create database connection") + } + var rawDB *gorm.DB + if err := withRetries(ctx, func(ctx context.Context) error { + var err error + // Need to give postgres dialect as otherwise gorm starts running + // in compatibility mode rawDB, err = gorm.Open("postgres", dbSQL) if err != nil { return retry.RetryableError(err) } return nil }); err != nil { - return fmt.Errorf("failed to connect to database: %w", err) + return fmt.Errorf("failed to initialize gorm: %w", err) } if rawDB == nil { - return fmt.Errorf("failed to create database connection") + return fmt.Errorf("failed to initialize gorm") } // Set connection configuration. From e1ab3ee64a5cd2fff58278efb864375185971b5f Mon Sep 17 00:00:00 2001 From: taddari Date: Tue, 8 Sep 2020 17:13:04 +0100 Subject: [PATCH 6/6] Changed formatting --- pkg/database/database.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 6f50ed218..a2e3c695c 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -145,27 +145,28 @@ func (db *Database) OpenWithCacher(ctx context.Context, cacher cache.Cacher) err // Establish a connection to the database. We use this later to register // opencenusus stats. - var dbSQL *sql.DB + var rawSQL *sql.DB if err := withRetries(ctx, func(ctx context.Context) error { var err error - dbSQL, err = sql.Open("ocsql", c.ConnectionString()) + rawSQL, err = sql.Open("ocsql", c.ConnectionString()) if err != nil { return retry.RetryableError(err) - } // enable periodic recording of sql.DBStats - db.statsCloser = ocsql.RecordStats(dbSQL, 5*time.Second) + } + db.statsCloser = ocsql.RecordStats(rawSQL, 5*time.Second) return nil }); err != nil { - return fmt.Errorf("failed to open connection to database: %w", err) + return fmt.Errorf("failed to create sql connection: %w", err) } - if dbSQL == nil { + if rawSQL == nil { return fmt.Errorf("failed to create database connection") } + var rawDB *gorm.DB if err := withRetries(ctx, func(ctx context.Context) error { var err error // Need to give postgres dialect as otherwise gorm starts running // in compatibility mode - rawDB, err = gorm.Open("postgres", dbSQL) + rawDB, err = gorm.Open("postgres", rawSQL) if err != nil { return retry.RetryableError(err) }