Skip to content

Commit b0ac950

Browse files
boxofradboxofrad
committed
Wait until bot identity has been renewed
1 parent 14ff176 commit b0ac950

17 files changed

+394
-254
lines changed

‎lib/tbot/loop.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ type runOnIntervalConfig struct {
7575
retryLimit int
7676
exitOnRetryExhausted bool
7777
waitBeforeFirstRun bool
78+
// identityReadyCh allows the service to wait until the internal bot identity
79+
// renewal has completed before running, to avoid spamming the logs if the
80+
// service doesn't support gracefully degrading when there is no API client
81+
// available.
82+
identityReadyCh <-chan struct{}
7883
}
7984

8085
// runOnInterval runs a function on a given interval, with retries and jitter.
@@ -97,6 +102,19 @@ func runOnInterval(ctx context.Context, cfg runOnIntervalConfig) error {
97102

98103
log := cfg.log.With("task", cfg.name)
99104

105+
if cfg.identityReadyCh != nil {
106+
select {
107+
case <-cfg.identityReadyCh:
108+
default:
109+
log.InfoContext(ctx, "Waiting for internal bot identity to be renewed before running")
110+
select {
111+
case <-cfg.identityReadyCh:
112+
case <-ctx.Done():
113+
return nil
114+
}
115+
}
116+
}
117+
100118
if cfg.clock == nil {
101119
cfg.clock = clockwork.NewRealClock()
102120
}

‎lib/tbot/service_application_output.go

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,14 @@ import (
3838
// ApplicationOutputService generates the artifacts necessary to connect to a
3939
// HTTP or TCP application using Teleport.
4040
type ApplicationOutputService struct {
41-
botAuthClient *apiclient.Client
42-
botCfg *config.BotConfig
43-
cfg *config.ApplicationOutput
44-
getBotIdentity getBotIdentityFn
45-
log *slog.Logger
46-
reloadBroadcaster *channelBroadcaster
47-
resolver reversetunnelclient.Resolver
41+
botAuthClient *apiclient.Client
42+
botIdentityReadyCh <-chan struct{}
43+
botCfg *config.BotConfig
44+
cfg *config.ApplicationOutput
45+
getBotIdentity getBotIdentityFn
46+
log *slog.Logger
47+
reloadBroadcaster *channelBroadcaster
48+
resolver reversetunnelclient.Resolver
4849
}
4950

5051
func (s *ApplicationOutputService) String() string {
@@ -60,13 +61,14 @@ func (s *ApplicationOutputService) Run(ctx context.Context) error {
6061
defer unsubscribe()
6162

6263
err := runOnInterval(ctx, runOnIntervalConfig{
63-
service: s.String(),
64-
name: "output-renewal",
65-
f: s.generate,
66-
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
67-
retryLimit: renewalRetryLimit,
68-
log: s.log,
69-
reloadCh: reloadCh,
64+
service: s.String(),
65+
name: "output-renewal",
66+
f: s.generate,
67+
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
68+
retryLimit: renewalRetryLimit,
69+
log: s.log,
70+
reloadCh: reloadCh,
71+
identityReadyCh: s.botIdentityReadyCh,
7072
})
7173
return trace.Wrap(err)
7274
}

‎lib/tbot/service_application_tunnel.go

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,14 @@ import (
4343
// an authenticating tunnel and will automatically issue and renew certificates
4444
// as needed.
4545
type ApplicationTunnelService struct {
46-
botCfg *config.BotConfig
47-
cfg *config.ApplicationTunnelService
48-
proxyPingCache *proxyPingCache
49-
log *slog.Logger
50-
resolver reversetunnelclient.Resolver
51-
botClient *apiclient.Client
52-
getBotIdentity getBotIdentityFn
46+
botCfg *config.BotConfig
47+
cfg *config.ApplicationTunnelService
48+
proxyPingCache *proxyPingCache
49+
log *slog.Logger
50+
resolver reversetunnelclient.Resolver
51+
botClient *apiclient.Client
52+
getBotIdentity getBotIdentityFn
53+
botIdentityReadyCh <-chan struct{}
5354
}
5455

5556
func (s *ApplicationTunnelService) Run(ctx context.Context) error {
@@ -118,6 +119,19 @@ func (s *ApplicationTunnelService) buildLocalProxyConfig(ctx context.Context) (l
118119
ctx, span := tracer.Start(ctx, "ApplicationTunnelService/buildLocalProxyConfig")
119120
defer span.End()
120121

122+
if s.botIdentityReadyCh != nil {
123+
select {
124+
case <-s.botIdentityReadyCh:
125+
default:
126+
s.log.InfoContext(ctx, "Waiting for internal bot identity to be renewed before running")
127+
select {
128+
case <-s.botIdentityReadyCh:
129+
case <-ctx.Done():
130+
return alpnproxy.LocalProxyConfig{}, nil
131+
}
132+
}
133+
}
134+
121135
// Determine the roles to use for the impersonated app access user. We fall
122136
// back to all the roles the bot has if none are configured.
123137
roles := s.cfg.Roles

‎lib/tbot/service_ca_rotation.go

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,10 +127,11 @@ const caRotationRetryBackoff = time.Second * 2
127127
// certificates issued by the old CA, and stop trusting the new CA.
128128
// - Update Servers -> Standby: So we can stop trusting the old CA.
129129
type caRotationService struct {
130-
log *slog.Logger
131-
reloadBroadcaster *channelBroadcaster
132-
botClient *apiclient.Client
133-
getBotIdentity getBotIdentityFn
130+
log *slog.Logger
131+
reloadBroadcaster *channelBroadcaster
132+
botClient *apiclient.Client
133+
getBotIdentity getBotIdentityFn
134+
botIdentityReadyCh <-chan struct{}
134135
}
135136

136137
func (s *caRotationService) String() string {
@@ -149,6 +150,19 @@ func (s *caRotationService) Run(ctx context.Context) error {
149150
}
150151
jitter := retryutils.DefaultJitter
151152

153+
if s.botIdentityReadyCh != nil {
154+
select {
155+
case <-s.botIdentityReadyCh:
156+
default:
157+
s.log.InfoContext(ctx, "Waiting for internal bot identity to be renewed before running")
158+
select {
159+
case <-s.botIdentityReadyCh:
160+
case <-ctx.Done():
161+
return nil
162+
}
163+
}
164+
}
165+
152166
for {
153167
err := s.watchCARotations(ctx, rd.attempt)
154168
if ctx.Err() != nil {

‎lib/tbot/service_client_credential.go

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,13 @@ type ClientCredentialOutputService struct {
3434
// botAuthClient should be an auth client using the bots internal identity.
3535
// This will not have any roles impersonated and should only be used to
3636
// fetch CAs.
37-
botAuthClient *apiclient.Client
38-
botCfg *config.BotConfig
39-
cfg *config.UnstableClientCredentialOutput
40-
getBotIdentity getBotIdentityFn
41-
log *slog.Logger
42-
reloadBroadcaster *channelBroadcaster
37+
botAuthClient *apiclient.Client
38+
botIdentityReadyCh <-chan struct{}
39+
botCfg *config.BotConfig
40+
cfg *config.UnstableClientCredentialOutput
41+
getBotIdentity getBotIdentityFn
42+
log *slog.Logger
43+
reloadBroadcaster *channelBroadcaster
4344
}
4445

4546
func (s *ClientCredentialOutputService) String() string {
@@ -55,13 +56,14 @@ func (s *ClientCredentialOutputService) Run(ctx context.Context) error {
5556
defer unsubscribe()
5657

5758
err := runOnInterval(ctx, runOnIntervalConfig{
58-
service: s.String(),
59-
name: "output-renewal",
60-
f: s.generate,
61-
interval: s.botCfg.CredentialLifetime.RenewalInterval,
62-
retryLimit: renewalRetryLimit,
63-
log: s.log,
64-
reloadCh: reloadCh,
59+
service: s.String(),
60+
name: "output-renewal",
61+
f: s.generate,
62+
interval: s.botCfg.CredentialLifetime.RenewalInterval,
63+
retryLimit: renewalRetryLimit,
64+
log: s.log,
65+
reloadCh: reloadCh,
66+
identityReadyCh: s.botIdentityReadyCh,
6567
})
6668
return trace.Wrap(err)
6769
}

‎lib/tbot/service_database_output.go

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,14 @@ import (
3939
// DatabaseOutputService generates the artifacts necessary to connect to a
4040
// database using Teleport.
4141
type DatabaseOutputService struct {
42-
botAuthClient *apiclient.Client
43-
botCfg *config.BotConfig
44-
cfg *config.DatabaseOutput
45-
getBotIdentity getBotIdentityFn
46-
log *slog.Logger
47-
reloadBroadcaster *channelBroadcaster
48-
resolver reversetunnelclient.Resolver
42+
botAuthClient *apiclient.Client
43+
botIdentityReadyCh <-chan struct{}
44+
botCfg *config.BotConfig
45+
cfg *config.DatabaseOutput
46+
getBotIdentity getBotIdentityFn
47+
log *slog.Logger
48+
reloadBroadcaster *channelBroadcaster
49+
resolver reversetunnelclient.Resolver
4950
}
5051

5152
func (s *DatabaseOutputService) String() string {
@@ -61,13 +62,14 @@ func (s *DatabaseOutputService) Run(ctx context.Context) error {
6162
defer unsubscribe()
6263

6364
err := runOnInterval(ctx, runOnIntervalConfig{
64-
service: s.String(),
65-
name: "output-renewal",
66-
f: s.generate,
67-
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
68-
retryLimit: renewalRetryLimit,
69-
log: s.log,
70-
reloadCh: reloadCh,
65+
service: s.String(),
66+
name: "output-renewal",
67+
f: s.generate,
68+
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
69+
retryLimit: renewalRetryLimit,
70+
log: s.log,
71+
reloadCh: reloadCh,
72+
identityReadyCh: s.botIdentityReadyCh,
7173
})
7274
return trace.Wrap(err)
7375
}

‎lib/tbot/service_database_tunnel.go

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,14 @@ func (a alpnProxyMiddleware) OnStart(ctx context.Context, lp *alpnproxy.LocalPro
6363
// connections to a remote database service. It is an authenticating tunnel and
6464
// will automatically issue and renew certificates as needed.
6565
type DatabaseTunnelService struct {
66-
botCfg *config.BotConfig
67-
cfg *config.DatabaseTunnelService
68-
proxyPingCache *proxyPingCache
69-
log *slog.Logger
70-
resolver reversetunnelclient.Resolver
71-
botClient *apiclient.Client
72-
getBotIdentity getBotIdentityFn
66+
botCfg *config.BotConfig
67+
cfg *config.DatabaseTunnelService
68+
proxyPingCache *proxyPingCache
69+
log *slog.Logger
70+
resolver reversetunnelclient.Resolver
71+
botClient *apiclient.Client
72+
getBotIdentity getBotIdentityFn
73+
botIdentityReadyCh <-chan struct{}
7374
}
7475

7576
// buildLocalProxyConfig initializes the service, fetching any initial information and setting
@@ -78,6 +79,19 @@ func (s *DatabaseTunnelService) buildLocalProxyConfig(ctx context.Context) (lpCf
7879
ctx, span := tracer.Start(ctx, "DatabaseTunnelService/buildLocalProxyConfig")
7980
defer span.End()
8081

82+
if s.botIdentityReadyCh != nil {
83+
select {
84+
case <-s.botIdentityReadyCh:
85+
default:
86+
s.log.InfoContext(ctx, "Waiting for internal bot identity to be renewed before running")
87+
select {
88+
case <-s.botIdentityReadyCh:
89+
case <-ctx.Done():
90+
return alpnproxy.LocalProxyConfig{}, nil
91+
}
92+
}
93+
}
94+
8195
// Determine the roles to use for the impersonated db access user. We fall
8296
// back to all the roles the bot has if none are configured.
8397
roles := s.cfg.Roles

‎lib/tbot/service_heartbeat.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ type heartbeatService struct {
4747
botCfg *config.BotConfig
4848
startedAt time.Time
4949
heartbeatSubmitter heartbeatSubmitter
50+
botIdentityReadyCh <-chan struct{}
5051
interval time.Duration
5152
retryLimit int
5253
}
@@ -117,6 +118,7 @@ func (s *heartbeatService) Run(ctx context.Context) error {
117118
isStartup = false
118119
return nil
119120
},
121+
identityReadyCh: s.botIdentityReadyCh,
120122
})
121123
return trace.Wrap(err)
122124
}

‎lib/tbot/service_identity_output.go

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,15 @@ type IdentityOutputService struct {
4848
// botAuthClient should be an auth client using the bots internal identity.
4949
// This will not have any roles impersonated and should only be used to
5050
// fetch CAs.
51-
botAuthClient *apiclient.Client
52-
botCfg *config.BotConfig
53-
cfg *config.IdentityOutput
54-
getBotIdentity getBotIdentityFn
55-
log *slog.Logger
56-
proxyPingCache *proxyPingCache
57-
reloadBroadcaster *channelBroadcaster
58-
resolver reversetunnelclient.Resolver
51+
botAuthClient *apiclient.Client
52+
botIdentityReadyCh <-chan struct{}
53+
botCfg *config.BotConfig
54+
cfg *config.IdentityOutput
55+
getBotIdentity getBotIdentityFn
56+
log *slog.Logger
57+
proxyPingCache *proxyPingCache
58+
reloadBroadcaster *channelBroadcaster
59+
resolver reversetunnelclient.Resolver
5960
// executablePath is called to get the path to the tbot executable.
6061
// Usually this is os.Executable
6162
executablePath func() (string, error)
@@ -75,13 +76,14 @@ func (s *IdentityOutputService) Run(ctx context.Context) error {
7576
defer unsubscribe()
7677

7778
err := runOnInterval(ctx, runOnIntervalConfig{
78-
service: s.String(),
79-
name: "output-renewal",
80-
f: s.generate,
81-
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
82-
retryLimit: renewalRetryLimit,
83-
log: s.log,
84-
reloadCh: reloadCh,
79+
service: s.String(),
80+
name: "output-renewal",
81+
f: s.generate,
82+
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
83+
retryLimit: renewalRetryLimit,
84+
log: s.log,
85+
reloadCh: reloadCh,
86+
identityReadyCh: s.botIdentityReadyCh,
8587
})
8688
return trace.Wrap(err)
8789
}

‎lib/tbot/service_kubernetes_output.go

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,15 @@ type KubernetesOutputService struct {
5353
// botAuthClient should be an auth client using the bots internal identity.
5454
// This will not have any roles impersonated and should only be used to
5555
// fetch CAs.
56-
botAuthClient *apiclient.Client
57-
botCfg *config.BotConfig
58-
cfg *config.KubernetesOutput
59-
getBotIdentity getBotIdentityFn
60-
log *slog.Logger
61-
proxyPingCache *proxyPingCache
62-
reloadBroadcaster *channelBroadcaster
63-
resolver reversetunnelclient.Resolver
56+
botAuthClient *apiclient.Client
57+
botIdentityReadyCh <-chan struct{}
58+
botCfg *config.BotConfig
59+
cfg *config.KubernetesOutput
60+
getBotIdentity getBotIdentityFn
61+
log *slog.Logger
62+
proxyPingCache *proxyPingCache
63+
reloadBroadcaster *channelBroadcaster
64+
resolver reversetunnelclient.Resolver
6465
// executablePath is called to get the path to the tbot executable.
6566
// Usually this is os.Executable
6667
executablePath func() (string, error)
@@ -79,13 +80,14 @@ func (s *KubernetesOutputService) Run(ctx context.Context) error {
7980
defer unsubscribe()
8081

8182
err := runOnInterval(ctx, runOnIntervalConfig{
82-
service: s.String(),
83-
name: "output-renewal",
84-
f: s.generate,
85-
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
86-
retryLimit: renewalRetryLimit,
87-
log: s.log,
88-
reloadCh: reloadCh,
83+
service: s.String(),
84+
name: "output-renewal",
85+
f: s.generate,
86+
interval: cmp.Or(s.cfg.CredentialLifetime, s.botCfg.CredentialLifetime).RenewalInterval,
87+
retryLimit: renewalRetryLimit,
88+
log: s.log,
89+
reloadCh: reloadCh,
90+
identityReadyCh: s.botIdentityReadyCh,
8991
})
9092
return trace.Wrap(err)
9193
}

0 commit comments

Comments
 (0)