Skip to content

Commit d002f02

Browse files
jgwestjgwest
committed
feat: support alternative temp directory to '/dev/shm', for generated manifests
Signed-off-by: Jonathan West <[email protected]>
1 parent adb68bc commit d002f02

12 files changed

+61
-31
lines changed

‎agent/main.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"text/tabwriter"
1515
"time"
1616

17+
"github.com/argoproj/gitops-engine/pkg/utils/io"
1718
"github.com/argoproj/gitops-engine/pkg/utils/text"
1819

1920
"github.com/go-logr/logr"
@@ -148,6 +149,7 @@ func newCmd(log logr.Logger) *cobra.Command {
148149

149150
StartProfiler(log)
150151
clusterCache := cache.NewClusterCache(config,
152+
io.TempPathUseDevShmIfAvailable(),
151153
cache.SetNamespaces(namespaces),
152154
cache.SetLogr(log),
153155
cache.SetPopulateResourceInfoHandler(func(un *unstructured.Unstructured, isRoot bool) (info interface{}, cacheManifest bool) {
@@ -159,7 +161,7 @@ func newCmd(log logr.Logger) *cobra.Command {
159161
return
160162
}),
161163
)
162-
gitOpsEngine := engine.NewEngine(config, clusterCache, engine.WithLogr(log))
164+
gitOpsEngine := engine.NewEngine(config, clusterCache, io.TempPathUseDevShmIfAvailable(), engine.WithLogr(log))
163165
checkError(err, log)
164166

165167
cleanup, err := gitOpsEngine.Run()

‎pkg/cache/cluster.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,10 @@ type WeightedSemaphore interface {
143143
type ListRetryFunc func(err error) bool
144144

145145
// NewClusterCache creates new instance of cluster cache
146-
func NewClusterCache(config *rest.Config, opts ...UpdateSettingsFunc) *clusterCache {
146+
// - config: cluster config configuration
147+
// - tmpManifestPath: path where files passed to kubectl code are stored, see 'pkg/utils/io/io.go' for details
148+
// - opts: additional configuration options.
149+
func NewClusterCache(config *rest.Config, tmpManifestPath string, opts ...UpdateSettingsFunc) *clusterCache {
147150
log := textlogger.NewLogger(textlogger.NewConfig())
148151
cache := &clusterCache{
149152
settings: Settings{ResourceHealthOverride: &noopSettings{}, ResourcesFilter: &noopSettings{}},
@@ -155,8 +158,9 @@ func NewClusterCache(config *rest.Config, opts ...UpdateSettingsFunc) *clusterCa
155158
nsIndex: make(map[string]map[kube.ResourceKey]*Resource),
156159
config: config,
157160
kubectl: &kube.KubectlCmd{
158-
Log: log,
159-
Tracer: tracing.NopTracer{},
161+
Log: log,
162+
Tracer: tracing.NopTracer{},
163+
TmpPath: tmpManifestPath,
160164
},
161165
syncStatus: clusterCacheSync{
162166
resyncTimeout: defaultClusterResyncTimeout,

‎pkg/cache/cluster_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
testcore "k8s.io/client-go/testing"
3131
"sigs.k8s.io/yaml"
3232

33+
"github.com/argoproj/gitops-engine/pkg/utils/io"
3334
"github.com/argoproj/gitops-engine/pkg/utils/kube"
3435
"github.com/argoproj/gitops-engine/pkg/utils/kube/kubetest"
3536
)
@@ -119,7 +120,7 @@ func newClusterWithOptions(t *testing.T, opts []UpdateSettingsFunc, objs ...runt
119120
}, opts...)
120121

121122
cache := NewClusterCache(
122-
&rest.Config{Host: "https://test"},
123+
&rest.Config{Host: "https://test"}, io.TempPathUseDevShmIfAvailable(),
123124
opts...,
124125
)
125126
return cache
@@ -779,7 +780,7 @@ func ExampleNewClusterCache_resourceUpdatedEvents() {
779780
// kubernetes cluster config here
780781
config := &rest.Config{}
781782

782-
clusterCache := NewClusterCache(config)
783+
clusterCache := NewClusterCache(config, io.TempPathUseDevShmIfAvailable())
783784
// Ensure cluster is synced before using it
784785
if err := clusterCache.EnsureSynced(); err != nil {
785786
panic(err)

‎pkg/cache/predicates_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"testing"
66

7+
"github.com/argoproj/gitops-engine/pkg/utils/io"
78
"github.com/argoproj/gitops-engine/pkg/utils/kube"
89

910
"github.com/stretchr/testify/assert"
@@ -96,6 +97,7 @@ func ExampleNewClusterCache_inspectNamespaceResources() {
9697

9798
clusterCache := NewClusterCache(config,
9899
// cache default namespace only
100+
io.TempPathUseDevShmIfAvailable(),
99101
SetNamespaces([]string{"default", "kube-system"}),
100102
// configure custom logic to cache resources manifest and additional metadata
101103
SetPopulateResourceInfoHandler(func(un *unstructured.Unstructured, isRoot bool) (info interface{}, cacheManifest bool) {

‎pkg/cache/resource_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ package cache
33
import (
44
"testing"
55

6+
"github.com/argoproj/gitops-engine/pkg/utils/io"
67
"github.com/stretchr/testify/assert"
78
"k8s.io/client-go/rest"
89
)
910

10-
var cacheTest = NewClusterCache(&rest.Config{})
11+
var cacheTest = NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable())
1112

1213
func TestIsParentOf(t *testing.T) {
1314
child := cacheTest.newResource(mustToUnstructured(testPod()))

‎pkg/cache/settings_test.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ import (
77
"github.com/stretchr/testify/assert"
88
"k8s.io/client-go/rest"
99

10+
"github.com/argoproj/gitops-engine/pkg/utils/io"
1011
"github.com/argoproj/gitops-engine/pkg/utils/kube/kubetest"
1112
)
1213

1314
func TestSetSettings(t *testing.T) {
14-
cache := NewClusterCache(&rest.Config{}, SetKubectl(&kubetest.MockKubectlCmd{}))
15+
cache := NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable(), SetKubectl(&kubetest.MockKubectlCmd{}))
1516
updatedHealth := &noopSettings{}
1617
updatedFilter := &noopSettings{}
1718
cache.Invalidate(SetSettings(Settings{ResourceHealthOverride: updatedHealth, ResourcesFilter: updatedFilter}))
@@ -21,15 +22,15 @@ func TestSetSettings(t *testing.T) {
2122
}
2223

2324
func TestSetConfig(t *testing.T) {
24-
cache := NewClusterCache(&rest.Config{}, SetKubectl(&kubetest.MockKubectlCmd{}))
25+
cache := NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable(), SetKubectl(&kubetest.MockKubectlCmd{}))
2526
updatedConfig := &rest.Config{Host: "http://newhost"}
2627
cache.Invalidate(SetConfig(updatedConfig))
2728

2829
assert.Equal(t, updatedConfig, cache.config)
2930
}
3031

3132
func TestSetNamespaces(t *testing.T) {
32-
cache := NewClusterCache(&rest.Config{}, SetKubectl(&kubetest.MockKubectlCmd{}), SetNamespaces([]string{"default"}))
33+
cache := NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable(), SetKubectl(&kubetest.MockKubectlCmd{}), SetNamespaces([]string{"default"}))
3334

3435
updatedNamespaces := []string{"updated"}
3536
cache.Invalidate(SetNamespaces(updatedNamespaces))
@@ -38,7 +39,7 @@ func TestSetNamespaces(t *testing.T) {
3839
}
3940

4041
func TestSetResyncTimeout(t *testing.T) {
41-
cache := NewClusterCache(&rest.Config{})
42+
cache := NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable())
4243
assert.Equal(t, defaultClusterResyncTimeout, cache.syncStatus.resyncTimeout)
4344

4445
timeout := 1 * time.Hour
@@ -48,10 +49,10 @@ func TestSetResyncTimeout(t *testing.T) {
4849
}
4950

5051
func TestSetWatchResyncTimeout(t *testing.T) {
51-
cache := NewClusterCache(&rest.Config{})
52+
cache := NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable())
5253
assert.Equal(t, defaultWatchResyncTimeout, cache.watchResyncTimeout)
5354

5455
timeout := 30 * time.Minute
55-
cache = NewClusterCache(&rest.Config{}, SetWatchResyncTimeout(timeout))
56+
cache = NewClusterCache(&rest.Config{}, io.TempPathUseDevShmIfAvailable(), SetWatchResyncTimeout(timeout))
5657
assert.Equal(t, timeout, cache.watchResyncTimeout)
5758
}

‎pkg/engine/engine.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,9 @@ type gitOpsEngine struct {
4646
}
4747

4848
// NewEngine creates new instances of the GitOps engine
49-
func NewEngine(config *rest.Config, clusterCache cache.ClusterCache, opts ...Option) GitOpsEngine {
50-
o := applyOptions(opts)
49+
// - tmpPath: path where files passed to kubectl code are stored, see 'pkg/utils/io/io.go' for details
50+
func NewEngine(config *rest.Config, clusterCache cache.ClusterCache, tmpPath string, opts ...Option) GitOpsEngine {
51+
o := applyOptions(opts, tmpPath)
5152
return &gitOpsEngine{
5253
config: config,
5354
cache: clusterCache,

‎pkg/engine/engine_options.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@ type options struct {
1515
kubectl kube.Kubectl
1616
}
1717

18-
func applyOptions(opts []Option) options {
18+
func applyOptions(opts []Option, tmpPath string) options {
1919
log := textlogger.NewLogger(textlogger.NewConfig())
2020
o := options{
2121
log: log,
2222
kubectl: &kube.KubectlCmd{
23-
Log: log,
24-
Tracer: tracing.NopTracer{},
23+
Log: log,
24+
Tracer: tracing.NopTracer{},
25+
TmpPath: tmpPath,
2526
},
2627
}
2728
for _, opt := range opts {

‎pkg/utils/io/io.go

+16-6
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,31 @@
11
package io
22

3-
import (
4-
"os"
5-
)
3+
import "os"
64

75
var (
8-
// TempDir is set to '/dev/shm' if exists, otherwise is "", which defaults to os.TempDir() when passed to os.CreateTemp()
9-
TempDir string
6+
// TempDir is set to '/dev/shm' if it exists, otherwise is "", which defaults to os.TempDir() when passed to os.CreateTemp()
7+
devShmTempPath string
108
)
119

1210
func init() {
1311
fileInfo, err := os.Stat("/dev/shm")
1412
if err == nil && fileInfo.IsDir() {
15-
TempDir = "/dev/shm"
13+
devShmTempPath = "/dev/shm"
1614
}
1715
}
1816

17+
// TempPathUseDevShmIfAvailable will return '/dev/shm' if it is available on the system, otherwise it will return "", which defaults to os.TempDir() when passed to os.CreateTemp()
18+
//
19+
// The result of this function is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests.
20+
//
21+
// NOTE: There are tradeoffs to using this function: '/dev/shm' is backed by RAM, and thus has limited size.
22+
// - Since it is backed by RAM, this has the advantage of ensuring that sensitive data (such as credentials) are kept off disk (absent disk caching of memory)
23+
// - However, due to the limited size, '/dev/shm' may run out of disk space, and/or is more vulnerable to slow leaks of files over time.
24+
// You may instead consider using a disk-backed storage path like "", which os.CreateTemp() will default to e.g. '/tmp'.
25+
func TempPathUseDevShmIfAvailable() string {
26+
return devShmTempPath
27+
}
28+
1929
// DeleteFile is best effort deletion of a file
2030
func DeleteFile(path string) {
2131
if _, err := os.Stat(path); os.IsNotExist(err) {

‎pkg/utils/kube/ctl.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ type KubectlCmd struct {
4545
Log logr.Logger
4646
Tracer tracing.Tracer
4747
OnKubectlRun OnKubectlRunFunc
48+
// TmpPath is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests. See 'utils/io/io.go' for details.
49+
TmpPath string
4850
}
4951

5052
type APIResourceInfo struct {
@@ -272,7 +274,7 @@ func (k *KubectlCmd) DeleteResource(ctx context.Context, config *rest.Config, gv
272274
}
273275

274276
func (k *KubectlCmd) ManageResources(config *rest.Config, openAPISchema openapi.Resources) (ResourceOperations, func(), error) {
275-
f, err := os.CreateTemp(utils.TempDir, "")
277+
f, err := os.CreateTemp(k.TmpPath, "")
276278
if err != nil {
277279
return nil, nil, fmt.Errorf("failed to generate temp file for kubeconfig: %v", err)
278280
}
@@ -293,6 +295,7 @@ func (k *KubectlCmd) ManageResources(config *rest.Config, openAPISchema openapi.
293295
tracer: k.Tracer,
294296
log: k.Log,
295297
onKubectlRun: k.OnKubectlRun,
298+
tmpPath: k.TmpPath,
296299
}, cleanup, nil
297300
}
298301

‎pkg/utils/kube/ctl_test.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/stretchr/testify/assert"
1111
"k8s.io/klog/v2/textlogger"
1212

13+
"github.com/argoproj/gitops-engine/pkg/utils/io"
1314
testingutils "github.com/argoproj/gitops-engine/pkg/utils/testing"
1415
"github.com/argoproj/gitops-engine/pkg/utils/tracing"
1516
)
@@ -20,8 +21,9 @@ var (
2021

2122
func TestConvertToVersion(t *testing.T) {
2223
kubectl := KubectlCmd{
23-
Log: textlogger.NewLogger(textlogger.NewConfig()),
24-
Tracer: tracing.NopTracer{},
24+
Log: textlogger.NewLogger(textlogger.NewConfig()),
25+
Tracer: tracing.NopTracer{},
26+
TmpPath: io.TempPathUseDevShmIfAvailable(),
2527
}
2628
t.Run("AppsDeployment", func(t *testing.T) {
2729
newObj, err := kubectl.ConvertToVersion(testingutils.UnstructuredFromFile("testdata/appsdeployment.yaml"), "apps", "v1")

‎pkg/utils/kube/resource_ops.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ type kubectlResourceOperations struct {
5252
onKubectlRun OnKubectlRunFunc
5353
fact cmdutil.Factory
5454
openAPISchema openapi.Resources
55+
// tmpPath is used to store temporary files that are passed to kubectl code. These temporary files are usually cluster credentials or kubernetes manifests. See 'utils/io/io.go' for details.
56+
tmpPath string
5557
}
5658

5759
type commandExecutor func(f cmdutil.Factory, ioStreams genericclioptions.IOStreams, fileName string) error
@@ -61,16 +63,16 @@ func (k *kubectlResourceOperations) runResourceCommand(ctx context.Context, obj
6163
if err != nil {
6264
return "", err
6365
}
64-
manifestFile, err := os.CreateTemp(io.TempDir, "")
66+
manifestFile, err := os.CreateTemp(k.tmpPath, "")
6567
if err != nil {
66-
return "", fmt.Errorf("Failed to generate temp file for manifest: %v", err)
68+
return "", fmt.Errorf("failed to generate temp file for manifest: %v", err)
6769
}
6870
defer io.DeleteFile(manifestFile.Name())
6971
if _, err = manifestFile.Write(manifestBytes); err != nil {
70-
return "", fmt.Errorf("Failed to write manifest: %v", err)
72+
return "", fmt.Errorf("failed to write manifest: %v", err)
7173
}
7274
if err = manifestFile.Close(); err != nil {
73-
return "", fmt.Errorf("Failed to close manifest: %v", err)
75+
return "", fmt.Errorf("failed to close manifest: %v", err)
7476
}
7577

7678
// log manifest

0 commit comments

Comments
 (0)