Skip to content

Commit ea8b71b

Browse files
committed
Add central LRU cache that adjusts to available memory
Hugo has always been a active user of in-memory caches, but before this commit we did nothing to control the memory usage. One failing example would be loading lots of big JSON data files and unmarshal them via `transform.Unmarshal`. This commit consolidates all these caches into one single LRU cache with an eviction strategy that also considers used vs. available memory. Hugo will try to limit its memory usage to 1/4 or total system memory, but this can be controlled with the `HUGO_MEMORYLIMIT` environment variable (a float value representing Gigabytes). A natural next step after this would be to use this cache for `.Content`. Fixes gohugoio#7425 Fixes gohugoio#7437 Fixes gohugoio#7436 Fixes gohugoio#7882 Updates gohugoio#7544
1 parent fdfa4a5 commit ea8b71b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+2256
-907
lines changed

cache/memcache/memcache.go

+506
Large diffs are not rendered by default.

cache/memcache/memcache_test.go

+178
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
// Copyright 2020 The Hugo Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package memcache
15+
16+
import (
17+
"fmt"
18+
"path/filepath"
19+
"sync"
20+
"testing"
21+
"time"
22+
23+
qt "github.com/frankban/quicktest"
24+
)
25+
26+
func TestCache(t *testing.T) {
27+
t.Parallel()
28+
c := qt.New(t)
29+
30+
cache := New(Config{})
31+
32+
counter := 0
33+
create := func() Entry {
34+
counter++
35+
return Entry{Value: counter}
36+
}
37+
38+
a := cache.GetOrCreatePartition("a", ClearNever)
39+
40+
for i := 0; i < 5; i++ {
41+
v1, err := a.GetOrCreate("a1", create)
42+
c.Assert(err, qt.IsNil)
43+
c.Assert(v1, qt.Equals, 1)
44+
v2, err := a.GetOrCreate("a2", create)
45+
c.Assert(err, qt.IsNil)
46+
c.Assert(v2, qt.Equals, 2)
47+
}
48+
49+
cache.Clear()
50+
51+
v3, err := a.GetOrCreate("a2", create)
52+
c.Assert(err, qt.IsNil)
53+
c.Assert(v3, qt.Equals, 3)
54+
}
55+
56+
func TestCacheConcurrent(t *testing.T) {
57+
t.Parallel()
58+
59+
c := qt.New(t)
60+
61+
var wg sync.WaitGroup
62+
63+
cache := New(Config{})
64+
65+
create := func(i int) func() Entry {
66+
return func() Entry {
67+
return Entry{Value: i}
68+
}
69+
}
70+
71+
for i := 0; i < 10; i++ {
72+
wg.Add(1)
73+
go func() {
74+
defer wg.Done()
75+
for j := 0; j < 100; j++ {
76+
id := fmt.Sprintf("id%d", j)
77+
v, err := cache.getOrCreate("a", id, create(j))
78+
c.Assert(err, qt.IsNil)
79+
c.Assert(v, qt.Equals, j)
80+
}
81+
}()
82+
}
83+
wg.Wait()
84+
}
85+
86+
func TestCacheMemStats(t *testing.T) {
87+
t.Parallel()
88+
c := qt.New(t)
89+
90+
cache := New(Config{
91+
ItemsToPrune: 10,
92+
CheckInterval: 500 * time.Millisecond,
93+
})
94+
95+
s := cache.stats
96+
97+
c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true)
98+
c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0))
99+
c.Assert(s.availableMemory > 0, qt.Equals, true)
100+
c.Assert(s.numItems, qt.Equals, uint64(0))
101+
102+
counter := 0
103+
create := func() Entry {
104+
counter++
105+
return Entry{Value: counter}
106+
}
107+
108+
for i := 1; i <= 20; i++ {
109+
_, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create)
110+
c.Assert(err, qt.IsNil)
111+
}
112+
113+
c.Assert(s.getNumItems(), qt.Equals, uint64(20))
114+
cache.cache.SetMaxSize(10)
115+
time.Sleep(time.Millisecond * 600)
116+
c.Assert(int(s.getNumItems()), qt.Equals, 10)
117+
118+
}
119+
120+
func TestSplitBasePathAndExt(t *testing.T) {
121+
t.Parallel()
122+
c := qt.New(t)
123+
124+
tests := []struct {
125+
path string
126+
a string
127+
b string
128+
}{
129+
{"a/b.json", "a", "json"},
130+
{"a/b/c/d.json", "a", "json"},
131+
}
132+
for i, this := range tests {
133+
msg := qt.Commentf("test %d", i)
134+
a, b := splitBasePathAndExt(this.path)
135+
136+
c.Assert(a, qt.Equals, this.a, msg)
137+
c.Assert(b, qt.Equals, this.b, msg)
138+
}
139+
140+
}
141+
142+
func TestCleanKey(t *testing.T) {
143+
c := qt.New(t)
144+
145+
c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js")
146+
c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js")
147+
c.Assert(CleanKey("a.js"), qt.Equals, "_root/a.js")
148+
c.Assert(CleanKey("b/a"), qt.Equals, "b/a.unkn")
149+
150+
}
151+
152+
func TestKeyValid(t *testing.T) {
153+
c := qt.New(t)
154+
155+
c.Assert(keyValid("a/b.j"), qt.Equals, true)
156+
c.Assert(keyValid("a/b."), qt.Equals, false)
157+
c.Assert(keyValid("a/b"), qt.Equals, false)
158+
c.Assert(keyValid("/a/b.txt"), qt.Equals, false)
159+
c.Assert(keyValid("a\\b.js"), qt.Equals, false)
160+
161+
}
162+
163+
func TestInsertKeyPathElement(t *testing.T) {
164+
c := qt.New(t)
165+
166+
c.Assert(InsertKeyPathElements("a/b.j", "en"), qt.Equals, "a/en/b.j")
167+
c.Assert(InsertKeyPathElements("a/b.j", "en", "foo"), qt.Equals, "a/en/foo/b.j")
168+
c.Assert(InsertKeyPathElements("a/b.j", "", "foo"), qt.Equals, "a/foo/b.j")
169+
170+
}
171+
172+
func TestShouldEvict(t *testing.T) {
173+
// TODO1 remove?
174+
//c := qt.New(t)
175+
176+
//fmt.Println("=>", CleanKey("kkk"))
177+
//c.Assert(shouldEvict("key", Entry{}, ClearNever, identity.NewPathIdentity(files.ComponentFolderAssets, "a/b/c.js")), qt.Equals, true)
178+
}

cache/namedmemcache/named_cache.go

-79
This file was deleted.

cache/namedmemcache/named_cache_test.go

-80
This file was deleted.

commands/commands.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
301301
cmd.Flags().BoolP("path-warnings", "", false, "print warnings on duplicate target paths etc.")
302302
cmd.Flags().StringVarP(&cc.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`")
303303
cmd.Flags().StringVarP(&cc.memprofile, "profile-mem", "", "", "write memory profile to `file`")
304-
cmd.Flags().BoolVarP(&cc.printm, "print-mem", "", false, "print memory usage to screen at intervals")
304+
cmd.Flags().BoolVarP(&cc.printm, "printMem", "", false, "print memory usage to screen at intervals")
305305
cmd.Flags().StringVarP(&cc.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`")
306306
cmd.Flags().StringVarP(&cc.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)")
307307

commands/hugo.go

+7-15
Original file line numberDiff line numberDiff line change
@@ -427,8 +427,14 @@ func (c *commandeer) initMemTicker() func() {
427427
quit := make(chan struct{})
428428
printMem := func() {
429429
var m runtime.MemStats
430+
var cacheDropped int
431+
h := c.hugo()
432+
if h != nil && h.MemCache != nil {
433+
cacheDropped = h.MemCache.GetDropped()
434+
}
435+
430436
runtime.ReadMemStats(&m)
431-
fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\n\n", formatByteCount(m.Alloc), formatByteCount(m.TotalAlloc), formatByteCount(m.Sys), m.NumGC)
437+
fmt.Printf("\n\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMemCacheDropped = %d\nConfiguredMemoryLimit = %v\n\n", helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, cacheDropped, helpers.FormatByteCount(config.GetMemoryLimit()))
432438

433439
}
434440

@@ -1209,17 +1215,3 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
12091215

12101216
return name
12111217
}
1212-
1213-
func formatByteCount(b uint64) string {
1214-
const unit = 1000
1215-
if b < unit {
1216-
return fmt.Sprintf("%d B", b)
1217-
}
1218-
div, exp := int64(unit), 0
1219-
for n := b / unit; n >= unit; n /= unit {
1220-
div *= unit
1221-
exp++
1222-
}
1223-
return fmt.Sprintf("%.1f %cB",
1224-
float64(b)/float64(div), "kMGTPE"[exp])
1225-
}

compare/compare.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
package compare
1515

1616
// Eqer can be used to determine if this value is equal to the other.
17-
// The semantics of equals is that the two value are interchangeable
17+
// The semantics of equals is that the two values are interchangeable
1818
// in the Hugo templates.
1919
type Eqer interface {
2020
Eq(other interface{}) bool

0 commit comments

Comments
 (0)