Skip to content

Commit 9a75778

Browse files
nopcodernopcoder
authored
Setting kv driver defaults as configuration (#4553)
1 parent 2a74d86 commit 9a75778

15 files changed

+89
-83
lines changed

‎pkg/config/defaults.go

+46-18
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,35 @@ import (
77
)
88

99
const (
10-
DatabaseTypeKey = "database.type"
11-
DefaultDatabaseType = "local"
10+
DatabaseTypeKey = "database.type"
11+
LocalDatabaseType = "local"
1212

13-
DatabaseKVLocalPath = "database.local.path"
14-
DefaultDatabaseLocalKVPath = "~/lakefs/metadata"
13+
DatabaseLocalPathKey = "database.local.path"
14+
DefaultDatabaseLocalPath = "~/lakefs/metadata"
1515

16-
BlockstoreTypeKey = "blockstore.type"
17-
DefaultBlockstoreType = "local"
16+
DatabaseLocalPrefetchSizeKey = "database.local.prefetch_size"
17+
DefaultDatabaseLocalPrefetchSize = 256
18+
19+
DatabaseDynamodbTableNameKey = "database.dynamodb.table_name"
20+
DefaultDatabaseDynamodbTableName = "kvstore"
21+
22+
DatabaseDynamodbReadCapacityUnitsKey = "database.dynamodb.read_capacity_units"
23+
DefaultDatabaseDynamodbReadCapacityUnits = 1000
24+
25+
DatabaseDynamodbWriteCapacityUnitsKey = "database.dynamodb.write_capacity_units"
26+
DefaultDatabaseDynamodbWriteCapacityUnits = 1000
27+
28+
DatabasePostgresMaxOpenConnectionsKey = "database.postgres.max_open_connections"
29+
DefaultDatabasePostgresMaxOpenConnections = 25
30+
31+
DatabasePostgresMaxIdleConnectionsKey = "database.postgres.max_idle_connections"
32+
DefaultDatabasePostgresMaxIdleConnections = 25
33+
34+
PostgresConnectionMaxLifetimeKey = "database.postgres.connection_max_lifetime"
35+
DefaultPostgresConnectionMaxLifetime = "5m"
36+
37+
BlockstoreTypeKey = "blockstore.type"
38+
LocalBlockstoreType = "local"
1839

1940
BlockstoreLocalPathKey = "blockstore.local.path"
2041
DefaultBlockstoreLocalPath = "~/lakefs/data/block"
@@ -88,8 +109,8 @@ const (
88109
LoggingFilesKeepKey = "logging.files_keep"
89110
LoggingAuditLogLevel = "logging.audit_log_level"
90111

91-
AuthEncryptSecretKey = "auth.encrypt.secret_key" // #nosec
92-
DefaultAuthEncryptSecretKey = "THIS_MUST_BE_CHANGED_IN_PRODUCTION" // #nosec
112+
AuthEncryptSecretKey = "auth.encrypt.secret_key" // #nosec
113+
LocalAuthEncryptSecretKey = "THIS_MUST_BE_CHANGED_IN_PRODUCTION" // #nosec
93114

94115
ActionsEnabledKey = "actions.enabled"
95116

@@ -134,17 +155,11 @@ const (
134155
UIEnabledKey = "ui.enabled"
135156
)
136157

137-
func setDefaultLocalConfig() {
138-
viper.SetDefault(DatabaseTypeKey, DefaultDatabaseType)
139-
viper.SetDefault(DatabaseKVLocalPath, DefaultDatabaseLocalKVPath)
140-
viper.SetDefault(BlockstoreLocalPathKey, DefaultBlockstoreLocalPath)
141-
viper.SetDefault(AuthEncryptSecretKey, DefaultAuthEncryptSecretKey)
142-
viper.SetDefault(BlockstoreTypeKey, DefaultBlockstoreType)
143-
}
144-
145158
func setDefaults(local bool) {
146159
if local {
147-
setDefaultLocalConfig()
160+
viper.SetDefault(DatabaseTypeKey, LocalDatabaseType)
161+
viper.SetDefault(AuthEncryptSecretKey, LocalAuthEncryptSecretKey)
162+
viper.SetDefault(BlockstoreTypeKey, LocalBlockstoreType)
148163
}
149164

150165
viper.SetDefault(ListenAddressKey, DefaultListenAddr)
@@ -166,7 +181,6 @@ func setDefaults(local bool) {
166181
viper.SetDefault(AuthLogoutRedirectURL, DefaultAuthLogoutRedirectURL)
167182

168183
viper.SetDefault(BlockstoreLocalPathKey, DefaultBlockstoreLocalPath)
169-
viper.SetDefault(BlockstoreTypeKey, DefaultBlockstoreType)
170184
viper.SetDefault(BlockstoreS3RegionKey, DefaultBlockstoreS3Region)
171185
viper.SetDefault(BlockstoreS3StreamingChunkSizeKey, DefaultBlockstoreS3StreamingChunkSize)
172186
viper.SetDefault(BlockstoreS3StreamingChunkTimeoutKey, DefaultBlockstoreS3StreamingChunkTimeout)
@@ -206,4 +220,18 @@ func setDefaults(local bool) {
206220
viper.SetDefault(LakefsEmailBaseURLKey, DefaultLakefsEmailBaseURL)
207221

208222
viper.SetDefault(UIEnabledKey, DefaultUIEnabled)
223+
224+
viper.SetDefault(BlockstoreLocalPathKey, DefaultBlockstoreLocalPath)
225+
226+
viper.SetDefault(DatabaseLocalPathKey, DefaultDatabaseLocalPath)
227+
viper.SetDefault(DatabaseLocalPrefetchSizeKey, DefaultDatabaseLocalPrefetchSize)
228+
229+
viper.SetDefault(DatabaseDynamodbTableNameKey, DefaultDatabaseDynamodbTableName)
230+
231+
viper.SetDefault(DatabaseDynamodbReadCapacityUnitsKey, DefaultDatabaseDynamodbReadCapacityUnits)
232+
viper.SetDefault(DatabaseDynamodbWriteCapacityUnitsKey, DefaultDatabaseDynamodbWriteCapacityUnits)
233+
234+
viper.SetDefault(DatabasePostgresMaxOpenConnectionsKey, DefaultDatabasePostgresMaxOpenConnections)
235+
viper.SetDefault(DatabasePostgresMaxIdleConnectionsKey, DefaultDatabasePostgresMaxIdleConnections)
236+
viper.SetDefault(PostgresConnectionMaxLifetimeKey, DefaultPostgresConnectionMaxLifetime)
209237
}

‎pkg/config/template.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ type configuration struct {
7777
// DropTables Development flag to delete tables after successful migration to KV
7878
DropTables bool `mapstructure:"drop_tables"`
7979
// Type Name of the KV Store driver DB implementation which is available according to the kv package Drivers function
80-
Type string `mapstructure:"type"`
80+
Type string `mapstructure:"type" validate:"required"`
8181

8282
Local *struct {
8383
// Path - Local directory path to store the DB files

‎pkg/config/testdata/aws_credentials.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
auth:
36
encrypt:
47
secret_key: "required in config"

‎pkg/config/testdata/aws_credentials_with_alias.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
auth:
36
encrypt:
47
secret_key: "required in config"

‎pkg/config/testdata/domain_name_prefix.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
---
2+
database:
3+
type: local
4+
5+
blockstore:
6+
type: local
7+
28
gateways:
39
s3:
410
domain_name:

‎pkg/config/testdata/valid_config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ logging:
55
output: "-"
66

77
database:
8+
type: postgres
89
postgres:
910
connection_string: test:///dev/null
1011
max_open_connections: 12

‎pkg/config/testdata/valid_custom_badger_config.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
logging:
36
format: text
47
level: NONE

‎pkg/config/testdata/valid_gs_adapter_config.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
logging:
36
format: text
47
level: NONE

‎pkg/config/testdata/valid_json_logger_config.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
logging:
36
format: json
47
level: DEBUG

‎pkg/config/testdata/valid_oidc_config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ logging:
55
output: "-"
66

77
database:
8+
type: postgres
89
postgres:
910
connection_string: test:///dev/null
1011
max_open_connections: 12

‎pkg/config/testdata/valid_s3_adapter_config.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
---
2+
database:
3+
type: local
4+
25
logging:
36
format: text
47
level: NONE

‎pkg/kv/dynamodb/store.go

-20
Original file line numberDiff line numberDiff line change
@@ -47,32 +47,13 @@ const (
4747
PartitionKey = "PartitionKey"
4848
ItemKey = "ItemKey"
4949
ItemValue = "ItemValue"
50-
51-
DefaultDynamoDBTableName = "kvstore"
52-
// TODO (niro): Which values to use for DynamoDB tables?
53-
DefaultDynamoDBReadCapacityUnits = 1000
54-
DefaultDynamoDBWriteCapacityUnits = 1000
5550
)
5651

5752
//nolint:gochecknoinits
5853
func init() {
5954
kv.Register(DriverName, &Driver{})
6055
}
6156

62-
func normalizeDBParams(p *kvparams.DynamoDB) {
63-
if len(p.TableName) == 0 {
64-
p.TableName = DefaultDynamoDBTableName
65-
}
66-
67-
if p.ReadCapacityUnits == 0 {
68-
p.ReadCapacityUnits = DefaultDynamoDBReadCapacityUnits
69-
}
70-
71-
if p.WriteCapacityUnits == 0 {
72-
p.WriteCapacityUnits = DefaultDynamoDBWriteCapacityUnits
73-
}
74-
}
75-
7657
// Open - opens and returns a KV store over DynamoDB. This function creates the DB session
7758
// and sets up the KV table.
7859
func (d *Driver) Open(ctx context.Context, kvParams kvparams.KV) (kv.Store, error) {
@@ -81,7 +62,6 @@ func (d *Driver) Open(ctx context.Context, kvParams kvparams.KV) (kv.Store, erro
8162
return nil, fmt.Errorf("missing %s settings: %w", DriverName, kv.ErrDriverConfiguration)
8263
}
8364

84-
normalizeDBParams(params)
8565
sess, err := session.NewSessionWithOptions(session.Options{
8666
SharedConfigState: session.SharedConfigEnable,
8767
Profile: params.AwsProfile,

‎pkg/kv/local/driver.go

+1-13
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,7 @@ import (
1212
)
1313

1414
const (
15-
DriverName = "local"
16-
DefaultDirectoryPath = "~/lakefs/metadata"
17-
DefaultPrefetchSize = 256
15+
DriverName = "local"
1816
)
1917

2018
var (
@@ -24,21 +22,11 @@ var (
2422

2523
type Driver struct{}
2624

27-
func normalizeDBParams(p *kvparams.Local) {
28-
if len(p.Path) == 0 {
29-
p.Path = DefaultDirectoryPath
30-
}
31-
if p.PrefetchSize <= 0 {
32-
p.PrefetchSize = DefaultPrefetchSize
33-
}
34-
}
35-
3625
func (d *Driver) Open(ctx context.Context, kvParams kvparams.KV) (kv.Store, error) {
3726
params := kvParams.Local
3827
if params == nil {
3928
return nil, fmt.Errorf("missing %s settings: %w", DriverName, kv.ErrDriverConfiguration)
4029
}
41-
normalizeDBParams(params)
4230

4331
driverLock.Lock()
4432
defer driverLock.Unlock()

‎pkg/kv/params/database.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ type Postgres struct {
2727
MaxOpenConnections int32
2828
MaxIdleConnections int32
2929
ConnectionMaxLifetime time.Duration
30-
ScanPageSize int32
30+
ScanPageSize int
3131
Metrics bool
3232
}
3333

‎pkg/kv/postgres/store.go

+14-30
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"errors"
66
"fmt"
77
"strconv"
8-
"time"
98

109
"github.com/IBM/pgxpoolprometheus"
1110
"github.com/georgysavva/scany/pgxscan"
@@ -42,37 +41,15 @@ const (
4241

4342
// DefaultPartitions Changing the below value means repartitioning and probably a migration.
4443
// Change it only if you really know what you're doing.
45-
DefaultPartitions = 100
46-
47-
DefaultMaxOpenConnections = 25
48-
DefaultMaxIdleConnections = 25
49-
DefaultConnectionMaxLifetime = 5 * time.Minute
50-
DefaultScanPageSize = 1000
44+
DefaultPartitions = 100
45+
DefaultScanPageSize = 1000
5146
)
5247

5348
//nolint:gochecknoinits
5449
func init() {
5550
kv.Register(DriverName, &Driver{})
5651
}
5752

58-
func normalizeDBParams(p *kvparams.Postgres) {
59-
if p.MaxOpenConnections == 0 {
60-
p.MaxOpenConnections = DefaultMaxOpenConnections
61-
}
62-
63-
if p.MaxIdleConnections == 0 {
64-
p.MaxIdleConnections = DefaultMaxIdleConnections
65-
}
66-
67-
if p.ConnectionMaxLifetime == 0 {
68-
p.ConnectionMaxLifetime = DefaultConnectionMaxLifetime
69-
}
70-
71-
if p.ScanPageSize == 0 {
72-
p.ScanPageSize = DefaultScanPageSize
73-
}
74-
}
75-
7653
func (d *Driver) Open(ctx context.Context, kvParams kvparams.KV) (kv.Store, error) {
7754
if kvParams.Postgres == nil {
7855
return nil, fmt.Errorf("missing %s settings: %w", DriverName, kv.ErrDriverConfiguration)
@@ -131,14 +108,19 @@ func (d *Driver) Open(ctx context.Context, kvParams kvparams.KV) (kv.Store, erro
131108
}
132109

133110
func newPgxpoolConfig(kvParams kvparams.KV) (*pgxpool.Config, error) {
134-
normalizeDBParams(kvParams.Postgres)
135111
config, err := pgxpool.ParseConfig(kvParams.Postgres.ConnectionString)
136112
if err != nil {
137113
return nil, fmt.Errorf("%w: %s", kv.ErrDriverConfiguration, err)
138114
}
139-
config.MaxConns = kvParams.Postgres.MaxOpenConnections
140-
config.MinConns = kvParams.Postgres.MaxIdleConnections
141-
config.MaxConnLifetime = kvParams.Postgres.ConnectionMaxLifetime
115+
if kvParams.Postgres.MaxOpenConnections > 0 {
116+
config.MaxConns = kvParams.Postgres.MaxOpenConnections
117+
}
118+
if kvParams.Postgres.MaxIdleConnections > 0 {
119+
config.MinConns = kvParams.Postgres.MaxIdleConnections
120+
}
121+
if kvParams.Postgres.ConnectionMaxLifetime > 0 {
122+
config.MaxConnLifetime = kvParams.Postgres.ConnectionMaxLifetime
123+
}
142124
return config, err
143125
}
144126

@@ -162,7 +144,9 @@ func parseStoreConfig(runtimeParams map[string]string, pgParams *kvparams.Postgr
162144
}
163145

164146
p.SanitizedTableName = pgx.Identifier{p.TableName}.Sanitize()
165-
p.ScanPageSize = int(pgParams.ScanPageSize)
147+
if pgParams.ScanPageSize > 0 {
148+
p.ScanPageSize = pgParams.ScanPageSize
149+
}
166150
return p
167151
}
168152

0 commit comments

Comments
 (0)