Skip to content

Commit cf42455

Browse files
twz123twz123
authored
Merge pull request #5487 from twz123/controllerworker-without-join-token
Don't use join tokens to bootstrap embedded kubelet
2 parents 3af69c2 + bf2a8e2 commit cf42455

File tree

8 files changed

+169
-144
lines changed

8 files changed

+169
-144
lines changed

‎cmd/controller/controller.go

+8-40
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ import (
3232
"syscall"
3333
"time"
3434

35-
"github.com/avast/retry-go"
3635
workercmd "github.com/k0sproject/k0s/cmd/worker"
3736
"github.com/k0sproject/k0s/internal/pkg/dir"
3837
"github.com/k0sproject/k0s/internal/pkg/file"
@@ -62,11 +61,14 @@ import (
6261
"github.com/k0sproject/k0s/pkg/performance"
6362
"github.com/k0sproject/k0s/pkg/telemetry"
6463
"github.com/k0sproject/k0s/pkg/token"
65-
"github.com/sirupsen/logrus"
66-
"github.com/spf13/cobra"
64+
6765
"k8s.io/apimachinery/pkg/fields"
6866
apitypes "k8s.io/apimachinery/pkg/types"
6967
"k8s.io/client-go/rest"
68+
69+
"github.com/avast/retry-go"
70+
"github.com/sirupsen/logrus"
71+
"github.com/spf13/cobra"
7072
)
7173

7274
type command config.CLIOptions
@@ -636,7 +638,7 @@ func (c *command) start(ctx context.Context, flags *config.ControllerOptions) er
636638

637639
if controllerMode.WorkloadsEnabled() {
638640
perfTimer.Checkpoint("starting-worker")
639-
if err := c.startWorker(ctx, nodeName, kubeletExtraArgs, flags, nodeConfig); err != nil {
641+
if err := c.startWorker(ctx, nodeName, kubeletExtraArgs, flags); err != nil {
640642
logrus.WithError(err).Error("Failed to start controller worker")
641643
} else {
642644
perfTimer.Checkpoint("started-worker")
@@ -655,48 +657,18 @@ func (c *command) start(ctx context.Context, flags *config.ControllerOptions) er
655657
return nil
656658
}
657659

658-
func (c *command) startWorker(ctx context.Context, nodeName apitypes.NodeName, kubeletExtraArgs stringmap.StringMap, opts *config.ControllerOptions, nodeConfig *v1beta1.ClusterConfig) error {
659-
var bootstrapConfig string
660-
if !file.Exists(c.K0sVars.KubeletAuthConfigPath) {
661-
// wait for controller to start up
662-
err := retry.Do(func() error {
663-
if !file.Exists(c.K0sVars.AdminKubeConfigPath) {
664-
return fmt.Errorf("file does not exist: %s", c.K0sVars.AdminKubeConfigPath)
665-
}
666-
return nil
667-
}, retry.Context(ctx))
668-
if err != nil {
669-
return err
670-
}
671-
672-
err = retry.Do(func() error {
673-
// five minutes here are coming from maximum theoretical duration of kubelet bootstrap process
674-
// we use retry.Do with 10 attempts, back-off delay and delay duration 500 ms which gives us
675-
// 225 seconds here
676-
tokenAge := time.Second * 225
677-
cfg, err := token.CreateKubeletBootstrapToken(ctx, nodeConfig.Spec.API, c.K0sVars, token.RoleWorker, tokenAge)
678-
if err != nil {
679-
return err
680-
}
681-
bootstrapConfig = cfg
682-
return nil
683-
}, retry.Context(ctx))
684-
if err != nil {
685-
return err
686-
}
687-
}
660+
func (c *command) startWorker(ctx context.Context, nodeName apitypes.NodeName, kubeletExtraArgs stringmap.StringMap, opts *config.ControllerOptions) error {
688661
// Cast and make a copy of the controller command so it can use the same
689662
// opts to start the worker. Needs to be a copy so the original token and
690663
// possibly other args won't get messed up.
691664
wc := workercmd.Command(*(*config.CLIOptions)(c))
692-
wc.TokenArg = bootstrapConfig
693665
wc.Labels = append(wc.Labels, fields.OneTermEqualSelector(constant.K0SNodeRoleLabel, "control-plane").String())
694666
if opts.Mode() == config.ControllerPlusWorkerMode && !opts.NoTaints {
695667
key := path.Join(constant.NodeRoleLabelNamespace, "master")
696668
taint := fields.OneTermEqualSelector(key, ":NoSchedule")
697669
wc.Taints = append(wc.Taints, taint.String())
698670
}
699-
return wc.Start(ctx, nodeName, kubeletExtraArgs, (*embeddingController)(opts))
671+
return wc.Start(ctx, nodeName, kubeletExtraArgs, kubernetes.KubeconfigFromFile(c.K0sVars.AdminKubeConfigPath), (*embeddingController)(opts))
700672
}
701673

702674
type embeddingController config.ControllerOptions
@@ -748,10 +720,6 @@ func joinController(ctx context.Context, tokenArg string, certRootDir string) (*
748720
return nil, fmt.Errorf("failed to create join client: %w", err)
749721
}
750722

751-
if joinClient.JoinTokenType() != "controller-bootstrap" {
752-
return nil, fmt.Errorf("wrong token type %s, expected type: controller-bootstrap", joinClient.JoinTokenType())
753-
}
754-
755723
logrus.Info("Joining existing cluster via ", joinClient.Address())
756724

757725
var caData v1beta1.CaResponse

‎cmd/token/preshared.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,10 @@ func createKubeConfig(tok *bootstraptokenv1.BootstrapTokenString, role, joinURL,
120120

121121
var userName string
122122
switch role {
123-
case "worker":
124-
userName = "kubelet-bootstrap"
125-
case "controller":
126-
userName = "controller-bootstrap"
123+
case token.RoleWorker:
124+
userName = token.WorkerTokenAuthName
125+
case token.RoleController:
126+
userName = token.ControllerTokenAuthName
127127
default:
128128
return fmt.Errorf("unknown role: %s", role)
129129
}

‎cmd/worker/worker.go

+82-6
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import (
2222
"fmt"
2323
"os"
2424
"os/signal"
25+
"path/filepath"
2526
"runtime"
2627
"syscall"
2728

29+
"github.com/k0sproject/k0s/internal/pkg/file"
2830
"github.com/k0sproject/k0s/internal/pkg/flags"
2931
internallog "github.com/k0sproject/k0s/internal/pkg/log"
3032
"github.com/k0sproject/k0s/internal/pkg/stringmap"
@@ -39,8 +41,11 @@ import (
3941
"github.com/k0sproject/k0s/pkg/config"
4042
"github.com/k0sproject/k0s/pkg/kubernetes"
4143
"github.com/k0sproject/k0s/pkg/node"
44+
"github.com/k0sproject/k0s/pkg/token"
4245

4346
apitypes "k8s.io/apimachinery/pkg/types"
47+
"k8s.io/client-go/tools/clientcmd"
48+
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
4449

4550
"github.com/sirupsen/logrus"
4651
"github.com/spf13/cobra"
@@ -83,8 +88,9 @@ func NewWorkerCmd() *cobra.Command {
8388
c.TokenArg = args[0]
8489
}
8590

86-
if c.TokenArg != "" && c.TokenFile != "" {
87-
return errors.New("you can only pass one token argument either as a CLI argument 'k0s worker [token]' or as a flag 'k0s worker --token-file [path]'")
91+
getBootstrapKubeconfig, err := kubeconfigGetterFromJoinToken(c.TokenFile, c.TokenArg)
92+
if err != nil {
93+
return err
8894
}
8995

9096
nodeName, kubeletExtraArgs, err := GetNodeName(&c.WorkerOptions)
@@ -104,7 +110,14 @@ func NewWorkerCmd() *cobra.Command {
104110
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
105111
defer cancel()
106112

107-
return c.Start(ctx, nodeName, kubeletExtraArgs, nil)
113+
// Check for legacy CA file (unused on worker-only nodes since 1.33)
114+
if legacyCAFile := filepath.Join(c.K0sVars.CertRootDir, "ca.crt"); file.Exists(legacyCAFile) {
115+
// Keep the file to allow interop between 1.32 and 1.33.
116+
// TODO automatically delete this file in future releases.
117+
logrus.Infof("The file %s is no longer used and can safely be deleted", legacyCAFile)
118+
}
119+
120+
return c.Start(ctx, nodeName, kubeletExtraArgs, getBootstrapKubeconfig, nil)
108121
},
109122
}
110123

@@ -136,10 +149,73 @@ func GetNodeName(opts *config.WorkerOptions) (apitypes.NodeName, stringmap.Strin
136149
return nodeName, kubeletExtraArgs, nil
137150
}
138151

152+
func kubeconfigGetterFromJoinToken(tokenFile, tokenArg string) (clientcmd.KubeconfigGetter, error) {
153+
if tokenArg != "" {
154+
if tokenFile != "" {
155+
return nil, errors.New("you can only pass one token argument either as a CLI argument 'k0s worker [token]' or as a flag 'k0s worker --token-file [path]'")
156+
}
157+
158+
kubeconfig, err := loadKubeconfigFromJoinToken(tokenArg)
159+
if err != nil {
160+
return nil, err
161+
}
162+
163+
return func() (*clientcmdapi.Config, error) {
164+
return kubeconfig, nil
165+
}, nil
166+
}
167+
168+
if tokenFile == "" {
169+
return nil, nil
170+
}
171+
172+
return func() (*clientcmdapi.Config, error) {
173+
return loadKubeconfigFromTokenFile(tokenFile)
174+
}, nil
175+
}
176+
177+
func loadKubeconfigFromJoinToken(tokenData string) (*clientcmdapi.Config, error) {
178+
decoded, err := token.DecodeJoinToken(tokenData)
179+
if err != nil {
180+
return nil, fmt.Errorf("failed to decode join token: %w", err)
181+
}
182+
183+
kubeconfig, err := clientcmd.Load(decoded)
184+
if err != nil {
185+
return nil, fmt.Errorf("failed to load kubeconfig from join token: %w", err)
186+
}
187+
188+
if tokenType := token.GetTokenType(kubeconfig); tokenType != "kubelet-bootstrap" {
189+
return nil, fmt.Errorf("wrong token type %s, expected type: kubelet-bootstrap", tokenType)
190+
}
191+
192+
return kubeconfig, nil
193+
}
194+
195+
func loadKubeconfigFromTokenFile(path string) (*clientcmdapi.Config, error) {
196+
var problem string
197+
tokenBytes, err := os.ReadFile(path)
198+
if errors.Is(err, os.ErrNotExist) {
199+
problem = "not found"
200+
} else if err != nil {
201+
return nil, fmt.Errorf("failed to read token file: %w", err)
202+
} else if len(tokenBytes) == 0 {
203+
problem = "is empty"
204+
}
205+
if problem != "" {
206+
return nil, fmt.Errorf("token file %q %s"+
207+
`: obtain a new token via "k0s token create ..." and store it in the file`+
208+
` or reinstall this node via "k0s install --force ..." or "k0sctl apply --force ..."`,
209+
path, problem)
210+
}
211+
212+
return loadKubeconfigFromJoinToken(string(tokenBytes))
213+
}
214+
139215
// Start starts the worker components based on the given [config.CLIOptions].
140-
func (c *Command) Start(ctx context.Context, nodeName apitypes.NodeName, kubeletExtraArgs stringmap.StringMap, controller EmbeddingController) error {
141-
if err := worker.BootstrapKubeletKubeconfig(ctx, c.K0sVars, nodeName, &c.WorkerOptions); err != nil {
142-
return err
216+
func (c *Command) Start(ctx context.Context, nodeName apitypes.NodeName, kubeletExtraArgs stringmap.StringMap, getBootstrapKubeconfig clientcmd.KubeconfigGetter, controller EmbeddingController) error {
217+
if err := worker.BootstrapKubeletClientConfig(ctx, c.K0sVars, nodeName, &c.WorkerOptions, getBootstrapKubeconfig); err != nil {
218+
return fmt.Errorf("failed to bootstrap kubelet client configuration: %w", err)
143219
}
144220

145221
kubeletKubeconfigPath := c.K0sVars.KubeletAuthConfigPath

‎pkg/component/worker/kubelet.go

+28-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"github.com/k0sproject/k0s/pkg/component/manager"
3636
"github.com/k0sproject/k0s/pkg/config"
3737
"github.com/k0sproject/k0s/pkg/constant"
38+
"github.com/k0sproject/k0s/pkg/kubernetes"
3839
"github.com/k0sproject/k0s/pkg/supervisor"
3940

4041
corev1 "k8s.io/api/core/v1"
@@ -219,8 +220,13 @@ func (k *Kubelet) writeKubeletConfig() error {
219220
return err
220221
}
221222

223+
caPath, err := k.getKubeletCAPath()
224+
if err != nil {
225+
return err
226+
}
227+
222228
config := k.Configuration.DeepCopy()
223-
config.Authentication.X509.ClientCAFile = filepath.Join(k.K0sVars.CertRootDir, "ca.crt")
229+
config.Authentication.X509.ClientCAFile = caPath
224230
config.VolumePluginDir = k.K0sVars.KubeletVolumePluginDir
225231
config.ResolverConfig = determineKubeletResolvConfPath()
226232
config.StaticPodURL = staticPodURL
@@ -251,6 +257,27 @@ func (k *Kubelet) writeKubeletConfig() error {
251257
return nil
252258
}
253259

260+
func (k *Kubelet) getKubeletCAPath() (string, error) {
261+
restConfig, err := kubernetes.ClientConfig(kubernetes.KubeconfigFromFile(k.Kubeconfig))
262+
if err != nil {
263+
return "", fmt.Errorf("failed to load kubelet kubeconfig: %w", err)
264+
}
265+
266+
if len(restConfig.CAData) > 0 {
267+
caPath := filepath.Join(k.K0sVars.RunDir, "kubelet", "ca.crt")
268+
if err := file.WriteContentAtomically(caPath, restConfig.CAData, constant.CertMode); err != nil {
269+
return "", fmt.Errorf("failed to write kubelet CA file: %w", err)
270+
}
271+
return caPath, nil
272+
}
273+
274+
if !file.Exists(restConfig.CAFile) {
275+
return "", fmt.Errorf("kubelet CA file doesn't exist: %s", restConfig.CAFile)
276+
}
277+
278+
return restConfig.CAFile, nil
279+
}
280+
254281
func parseTaint(st string) (corev1.Taint, error) {
255282
var taint corev1.Taint
256283

0 commit comments

Comments
 (0)