Skip to content

Commit 933a476

Browse files
authored
Update goruntime to latest, 0.2.5. Add new config for watching changes in runtime config folder directly instead of the runtime root dir. (#151)
Signed-off-by: Yuki Sawa <[email protected]>
1 parent e406360 commit 933a476

File tree

11 files changed

+183
-15
lines changed

11 files changed

+183
-15
lines changed

README.md

+6
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,12 @@ RUNTIME_IGNOREDOTFILES default:"false"
320320

321321
**Configuration files are loaded from RUNTIME_ROOT/RUNTIME_SUBDIRECTORY/config/\*.yaml**
322322

323+
There are two methods for triggering a configuration reload:
324+
1. Symlink RUNTIME_ROOT to a different directory.
325+
2. Update the contents inside `RUNTIME_ROOT/RUNTIME_SUBDIRECTORY/config/` directly.
326+
327+
The former is the default behavior. To use the latter method, set the `RUNTIME_WATCH_ROOT` environment variable to `false`.
328+
323329
For more information on how runtime works you can read its [README](https://github.com/lyft/goruntime).
324330

325331
# Request Fields

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ require (
1313
github.com/gorilla/mux v1.7.4-0.20191121170500-49c01487a141
1414
github.com/kavu/go_reuseport v1.2.0
1515
github.com/kelseyhightower/envconfig v1.1.0
16-
github.com/lyft/goruntime v0.2.1
16+
github.com/lyft/goruntime v0.2.5
1717
github.com/lyft/gostats v0.4.0
1818
github.com/lyft/protoc-gen-validate v0.0.7-0.20180626203901-f9d2b11e4414 // indirect
1919
github.com/mediocregopher/radix/v3 v3.5.1

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ github.com/kelseyhightower/envconfig v1.1.0 h1:4htXR8ameS6KBfrNBoqEgpg0IK2D6rozN
4343
github.com/kelseyhightower/envconfig v1.1.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
4444
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
4545
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
46-
github.com/lyft/goruntime v0.2.1 h1:7DebA8oMVuoQ5TQ0j1xR/X2xRagbGrm0e2SoMdt5tRs=
47-
github.com/lyft/goruntime v0.2.1/go.mod h1:8rUh5gwIPQtyIkIXHbLN1j45HOb8cMgDhrw5GA7DF4g=
46+
github.com/lyft/goruntime v0.2.5 h1:yRmwOXl3Zns3+Z03fDMWt5+p609rfhIErh7HYCayODg=
47+
github.com/lyft/goruntime v0.2.5/go.mod h1:8rUh5gwIPQtyIkIXHbLN1j45HOb8cMgDhrw5GA7DF4g=
4848
github.com/lyft/gostats v0.4.0 h1:PbRWmwidTPk6Y80S6itBWDa+XVt1hGvqFM88TBJYdOo=
4949
github.com/lyft/gostats v0.4.0/go.mod h1:Tpx2xRzz4t+T2Tx0xdVgIoBdR2UMVz+dKnE3X01XSd8=
5050
github.com/lyft/protoc-gen-validate v0.0.7-0.20180626203901-f9d2b11e4414 h1:kLCSHuk3X+SI8Up26wM71id7jz77B3zCZDp01UWMVbM=

src/server/server_impl.go

+17-6
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"io"
88
"net/http"
99
"net/http/pprof"
10+
"path/filepath"
1011
"sort"
1112

1213
"os"
@@ -187,12 +188,22 @@ func newServer(name string, store stats.Store, localCache *freecache.Cache, opts
187188
loaderOpts = append(loaderOpts, loader.AllowDotFiles)
188189
}
189190

190-
ret.runtime = loader.New(
191-
s.RuntimePath,
192-
s.RuntimeSubdirectory,
193-
ret.store.Scope("runtime"),
194-
&loader.SymlinkRefresher{RuntimePath: s.RuntimePath},
195-
loaderOpts...)
191+
if s.RuntimeWatchRoot {
192+
ret.runtime = loader.New(
193+
s.RuntimePath,
194+
s.RuntimeSubdirectory,
195+
ret.store.Scope("runtime"),
196+
&loader.SymlinkRefresher{RuntimePath: s.RuntimePath},
197+
loaderOpts...)
198+
199+
} else {
200+
ret.runtime = loader.New(
201+
filepath.Join(s.RuntimePath, s.RuntimeSubdirectory),
202+
"config",
203+
ret.store.Scope("runtime"),
204+
&loader.DirectoryRefresher{},
205+
loaderOpts...)
206+
}
196207

197208
// setup http router
198209
ret.router = mux.NewRouter()

src/service/ratelimit.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ type service struct {
5757
stats serviceStats
5858
rlStatsScope stats.Scope
5959
legacy *legacyService
60+
runtimeWatchRoot bool
6061
}
6162

6263
func (this *service) reloadConfig() {
@@ -75,7 +76,7 @@ func (this *service) reloadConfig() {
7576
files := []config.RateLimitConfigToLoad{}
7677
snapshot := this.runtime.Snapshot()
7778
for _, key := range snapshot.Keys() {
78-
if !strings.HasPrefix(key, "config.") {
79+
if this.runtimeWatchRoot && !strings.HasPrefix(key, "config.") {
7980
continue
8081
}
8182

@@ -176,7 +177,7 @@ func (this *service) GetCurrentConfig() config.RateLimitConfig {
176177
}
177178

178179
func NewService(runtime loader.IFace, cache limiter.RateLimitCache,
179-
configLoader config.RateLimitConfigLoader, stats stats.Scope) RateLimitServiceServer {
180+
configLoader config.RateLimitConfigLoader, stats stats.Scope, runtimeWatchRoot bool) RateLimitServiceServer {
180181

181182
newService := &service{
182183
runtime: runtime,
@@ -187,6 +188,7 @@ func NewService(runtime loader.IFace, cache limiter.RateLimitCache,
187188
cache: cache,
188189
stats: newServiceStats(stats),
189190
rlStatsScope: stats.Scope("rate_limit"),
191+
runtimeWatchRoot: runtimeWatchRoot,
190192
}
191193
newService.legacy = &legacyService{
192194
s: newService,

src/service_cmd/runner/runner.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ func (runner *Runner) Run() {
6060
rand.New(limiter.NewLockedSource(time.Now().Unix())),
6161
s.ExpirationJitterMaxSeconds),
6262
config.NewRateLimitConfigLoaderImpl(),
63-
srv.Scope().Scope("service"))
63+
srv.Scope().Scope("service"),
64+
s.RuntimeWatchRoot,
65+
)
6466

6567
srv.AddDebugHttpEndpoint(
6668
"/rlconfig",

src/settings/settings.go

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ type Settings struct {
2020
RuntimePath string `envconfig:"RUNTIME_ROOT" default:"/srv/runtime_data/current"`
2121
RuntimeSubdirectory string `envconfig:"RUNTIME_SUBDIRECTORY"`
2222
RuntimeIgnoreDotFiles bool `envconfig:"RUNTIME_IGNOREDOTFILES" default:"false"`
23+
RuntimeWatchRoot bool `envconfig:"RUNTIME_WATCH_ROOT" default:"true"`
2324
LogLevel string `envconfig:"LOG_LEVEL" default:"WARN"`
2425
RedisSocketType string `envconfig:"REDIS_SOCKET_TYPE" default:"unix"`
2526
RedisUrl string `envconfig:"REDIS_URL" default:"/var/run/nutcracker/ratelimit.sock"`

test/integration/integration_test.go

+130
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"bytes"
77
"fmt"
88
"io/ioutil"
9+
"io"
910
"math/rand"
1011
"net/http"
1112
"os"
@@ -67,6 +68,11 @@ func TestBasicAuthConfig(t *testing.T) {
6768
t.Run("WithPerSecondRedisAuthWithLocalCache", testBasicConfigAuth("18093", "true", "1000"))
6869
}
6970

71+
func TestBasicReloadConfig(t *testing.T) {
72+
t.Run("BasicWithoutWatchRoot", testBasicConfigWithoutWatchRoot("8095", "false", "0"))
73+
t.Run("ReloadWithoutWatchRoot", testBasicConfigReload("8097", "false", "0", "false"))
74+
}
75+
7076
func testBasicConfigAuthTLS(grpcPort, perSecond string, local_cache_size string) func(*testing.T) {
7177
os.Setenv("REDIS_PERSECOND_URL", "localhost:16382")
7278
os.Setenv("REDIS_URL", "localhost:16381")
@@ -97,6 +103,28 @@ func testBasicConfigAuth(grpcPort, perSecond string, local_cache_size string) fu
97103
return testBasicBaseConfig(grpcPort, perSecond, local_cache_size)
98104
}
99105

106+
func testBasicConfigWithoutWatchRoot(grpcPort, perSecond string, local_cache_size string) func(*testing.T) {
107+
os.Setenv("REDIS_PERSECOND_URL", "localhost:6380")
108+
os.Setenv("REDIS_URL", "localhost:6379")
109+
os.Setenv("REDIS_AUTH", "")
110+
os.Setenv("REDIS_TLS", "false")
111+
os.Setenv("REDIS_PERSECOND_AUTH", "")
112+
os.Setenv("REDIS_PERSECOND_TLS", "false")
113+
os.Setenv("RUNTIME_WATCH_ROOT", "false")
114+
return testBasicBaseConfig(grpcPort, perSecond, local_cache_size)
115+
}
116+
117+
func testBasicConfigReload(grpcPort, perSecond string, local_cache_size, runtimeWatchRoot string) func(*testing.T) {
118+
os.Setenv("REDIS_PERSECOND_URL", "localhost:6380")
119+
os.Setenv("REDIS_URL", "localhost:6379")
120+
os.Setenv("REDIS_AUTH", "")
121+
os.Setenv("REDIS_TLS", "false")
122+
os.Setenv("REDIS_PERSECOND_AUTH", "")
123+
os.Setenv("REDIS_PERSECOND_TLS", "false")
124+
os.Setenv("RUNTIME_WATCH_ROOT", runtimeWatchRoot)
125+
return testConfigReload(grpcPort, perSecond, local_cache_size)
126+
}
127+
100128
func getCacheKey(cacheKey string, enableLocalCache bool) string {
101129
if enableLocalCache {
102130
return cacheKey + "_local"
@@ -456,3 +484,105 @@ func TestBasicConfigLegacy(t *testing.T) {
456484
assert.NoError(err)
457485
}
458486
}
487+
488+
func testConfigReload(grpcPort, perSecond string, local_cache_size string) func(*testing.T) {
489+
return func(t *testing.T) {
490+
os.Setenv("REDIS_PERSECOND", perSecond)
491+
os.Setenv("PORT", "8082")
492+
os.Setenv("GRPC_PORT", grpcPort)
493+
os.Setenv("DEBUG_PORT", "8084")
494+
os.Setenv("RUNTIME_ROOT", "runtime/current")
495+
os.Setenv("RUNTIME_SUBDIRECTORY", "ratelimit")
496+
os.Setenv("REDIS_PERSECOND_SOCKET_TYPE", "tcp")
497+
os.Setenv("REDIS_SOCKET_TYPE", "tcp")
498+
os.Setenv("LOCAL_CACHE_SIZE_IN_BYTES", local_cache_size)
499+
os.Setenv("USE_STATSD", "false")
500+
501+
local_cache_size_val, _ := strconv.Atoi(local_cache_size)
502+
enable_local_cache := local_cache_size_val > 0
503+
runner := runner.NewRunner()
504+
505+
go func() {
506+
runner.Run()
507+
}()
508+
509+
// HACK: Wait for the server to come up. Make a hook that we can wait on.
510+
time.Sleep(1 * time.Second)
511+
512+
assert := assert.New(t)
513+
conn, err := grpc.Dial(fmt.Sprintf("localhost:%s", grpcPort), grpc.WithInsecure())
514+
assert.NoError(err)
515+
defer conn.Close()
516+
c := pb.NewRateLimitServiceClient(conn)
517+
518+
response, err := c.ShouldRateLimit(
519+
context.Background(),
520+
common.NewRateLimitRequest("reload", [][][2]string{{{getCacheKey("block", enable_local_cache), "foo"}}}, 1))
521+
assert.Equal(
522+
&pb.RateLimitResponse{
523+
OverallCode: pb.RateLimitResponse_OK,
524+
Statuses: []*pb.RateLimitResponse_DescriptorStatus{{Code: pb.RateLimitResponse_OK}}},
525+
response)
526+
assert.NoError(err)
527+
528+
runner.GetStatsStore().Flush()
529+
loadCount1 := runner.GetStatsStore().NewCounter("ratelimit.service.config_load_success").Value()
530+
531+
// Copy a new file to config folder to test config reload functionality
532+
in, err := os.Open("runtime/current/ratelimit/reload.yaml")
533+
if err != nil {
534+
panic(err)
535+
}
536+
defer in.Close()
537+
out, err := os.Create("runtime/current/ratelimit/config/reload.yaml")
538+
if err != nil {
539+
panic(err)
540+
}
541+
defer out.Close()
542+
_, err = io.Copy(out, in)
543+
if err != nil {
544+
panic(err)
545+
}
546+
err = out.Close()
547+
if err != nil {
548+
panic(err)
549+
}
550+
551+
// Need to wait for config reload to take place and new descriptors to be loaded.
552+
// Shouldn't take more than 5 seconds but wait 120 at most just to be safe.
553+
wait := 120
554+
reloaded := false
555+
loadCount2 := uint64(0)
556+
557+
for i := 0; i < wait; i++ {
558+
time.Sleep(1 * time.Second)
559+
runner.GetStatsStore().Flush()
560+
loadCount2 = runner.GetStatsStore().NewCounter("ratelimit.service.config_load_success").Value()
561+
562+
// Check that successful loads count has increased before continuing.
563+
if loadCount2 > loadCount1 {
564+
reloaded = true
565+
break
566+
}
567+
}
568+
569+
assert.True(reloaded)
570+
assert.Greater(loadCount2, loadCount1)
571+
572+
response, err = c.ShouldRateLimit(
573+
context.Background(),
574+
common.NewRateLimitRequest("reload", [][][2]string{{{getCacheKey("key1", enable_local_cache), "foo"}}}, 1))
575+
assert.Equal(
576+
&pb.RateLimitResponse{
577+
OverallCode: pb.RateLimitResponse_OK,
578+
Statuses: []*pb.RateLimitResponse_DescriptorStatus{
579+
newDescriptorStatus(pb.RateLimitResponse_OK, 50, pb.RateLimitResponse_RateLimit_SECOND, 49)}},
580+
response)
581+
assert.NoError(err)
582+
583+
err = os.Remove("runtime/current/ratelimit/config/reload.yaml")
584+
if err != nil {
585+
panic(err)
586+
}
587+
}
588+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
domain: reload
2+
descriptors:
3+
- key: key1
4+
rate_limit:
5+
unit: second
6+
requests_per_unit: 50
7+
8+
- key: block
9+
rate_limit:
10+
unit: second
11+
requests_per_unit: 0
12+
13+
- key: one_per_minute
14+
rate_limit:
15+
unit: minute
16+
requests_per_unit: 1

test/service/ratelimit_legacy_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func TestInitialLoadErrorLegacy(test *testing.T) {
224224
func([]config.RateLimitConfigToLoad, stats.Scope) {
225225
panic(config.RateLimitConfigError("load error"))
226226
})
227-
service := ratelimit.NewService(t.runtime, t.cache, t.configLoader, t.statStore)
227+
service := ratelimit.NewService(t.runtime, t.cache, t.configLoader, t.statStore, true)
228228

229229
request := common.NewRateLimitRequestLegacy("test-domain", [][][2]string{{{"hello", "world"}}}, 1)
230230
response, err := service.GetLegacyService().ShouldRateLimit(nil, request)

test/service/ratelimit_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ func (this *rateLimitServiceTestSuite) setupBasicService() ratelimit.RateLimitSe
8282
this.configLoader.EXPECT().Load(
8383
[]config.RateLimitConfigToLoad{{"config.basic_config", "fake_yaml"}},
8484
gomock.Any()).Return(this.config)
85-
return ratelimit.NewService(this.runtime, this.cache, this.configLoader, this.statStore)
85+
return ratelimit.NewService(this.runtime, this.cache, this.configLoader, this.statStore, true)
8686
}
8787

8888
func TestService(test *testing.T) {
@@ -225,7 +225,7 @@ func TestInitialLoadError(test *testing.T) {
225225
func([]config.RateLimitConfigToLoad, stats.Scope) {
226226
panic(config.RateLimitConfigError("load error"))
227227
})
228-
service := ratelimit.NewService(t.runtime, t.cache, t.configLoader, t.statStore)
228+
service := ratelimit.NewService(t.runtime, t.cache, t.configLoader, t.statStore, true)
229229

230230
request := common.NewRateLimitRequest("test-domain", [][][2]string{{{"hello", "world"}}}, 1)
231231
response, err := service.ShouldRateLimit(nil, request)

0 commit comments

Comments
 (0)