Skip to content

Commit 9f16513

Browse files
nshankar13nshankar13
committed
[backport] cherry-pick 68e99eb to release-v1.1
Remove crd conversion webhook (openservicemesh#5065) Signed-off-by: nshankar13 <[email protected]> Add health handler Signed-off-by: nshankar13 <[email protected]>
1 parent fff4b0c commit 9f16513

31 files changed

+105
-1099
lines changed

‎charts/osm/templates/osm-bootstrap-deployment.yaml

-12
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,6 @@ spec:
5353
{{- if not (.Capabilities.APIVersions.Has "security.openshift.io/v1") }}
5454
{{- include "restricted.securityContext" . | nindent 6 }}
5555
{{- end }}
56-
initContainers:
57-
- name: init-osm-bootstrap
58-
image: "{{ include "osmCRDs.image" . }}"
59-
imagePullPolicy: {{ .Values.osm.image.pullPolicy }}
60-
command:
61-
- sh
62-
- -c
63-
- >
64-
kubectl apply -f /osm-crds;
65-
{{- if .Values.osm.enableReconciler }}
66-
kubectl label -f /osm-crds openservicemesh.io/reconcile=true --overwrite;
67-
{{- end }}
6856
containers:
6957
- name: osm-bootstrap
7058
image: "{{ include "osmBootstrap.image" . }}"

‎cmd/osm-bootstrap/crds/config_meshconfig.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- meshconfig
3131
singular: meshconfig
3232
plural: meshconfigs
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha2
3537
served: true

‎cmd/osm-bootstrap/crds/policy_egress.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- egress
3131
singular: egress
3232
plural: egresses
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha1
3537
served: true

‎cmd/osm-bootstrap/crds/policy_ingress_backend.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- ingressbackend
3131
singular: ingressbackend
3232
plural: ingressbackends
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha1
3537
served: true

‎cmd/osm-bootstrap/crds/policy_retry.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- retry
3131
singular: retry
3232
plural: retries
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha1
3537
served: true

‎cmd/osm-bootstrap/crds/policy_upstream_traffic_setting.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- upstreamtrafficsetting
3131
singular: upstreamtrafficsetting
3232
plural: upstreamtrafficsettings
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha1
3537
served: true

‎cmd/osm-bootstrap/crds/smi_http_route_group.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ spec:
2929
- htr
3030
plural: httproutegroups
3131
singular: httproutegroup
32+
conversion:
33+
strategy: None
3234
versions:
3335
- name: v1alpha4
3436
served: true

‎cmd/osm-bootstrap/crds/smi_tcp_route.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ spec:
2929
- tr
3030
plural: tcproutes
3131
singular: tcproute
32+
conversion:
33+
strategy: None
3234
versions:
3335
- name: v1alpha4
3436
served: true

‎cmd/osm-bootstrap/crds/smi_traffic_access.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ spec:
2929
- tt
3030
plural: traffictargets
3131
singular: traffictarget
32+
conversion:
33+
strategy: None
3234
versions:
3335
- name: v1alpha3
3436
served: true

‎cmd/osm-bootstrap/crds/smi_traffic_split.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- ts
3131
plural: trafficsplits
3232
singular: trafficsplit
33+
conversion:
34+
strategy: None
3335
versions:
3436
- name: v1alpha4
3537
served: false

‎cmd/osm-bootstrap/osm-bootstrap.go

+86-36
Original file line numberDiff line numberDiff line change
@@ -9,34 +9,36 @@ import (
99
"flag"
1010
"fmt"
1111
"os"
12+
"path/filepath"
13+
"strconv"
14+
"net/http"
1215

1316
"github.com/pkg/errors"
1417
"github.com/spf13/pflag"
1518
admissionv1 "k8s.io/api/admissionregistration/v1"
1619
corev1 "k8s.io/api/core/v1"
20+
apiv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1721
clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
1822
apiclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
1923
apierrors "k8s.io/apimachinery/pkg/api/errors"
2024
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2125
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2226
"k8s.io/apimachinery/pkg/runtime"
27+
"k8s.io/apimachinery/pkg/runtime/serializer"
2328
"k8s.io/client-go/kubernetes"
2429
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
2530
"k8s.io/client-go/tools/clientcmd"
2631
"k8s.io/kubectl/pkg/util"
2732

2833
configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
34+
configClientset "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned"
2935

3036
"github.com/openservicemesh/osm/pkg/certificate/providers"
31-
"github.com/openservicemesh/osm/pkg/configurator"
3237
"github.com/openservicemesh/osm/pkg/constants"
33-
"github.com/openservicemesh/osm/pkg/crdconversion"
34-
configClientset "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned"
3538
"github.com/openservicemesh/osm/pkg/httpserver"
3639
httpserverconstants "github.com/openservicemesh/osm/pkg/httpserver/constants"
3740
"github.com/openservicemesh/osm/pkg/k8s/events"
3841
"github.com/openservicemesh/osm/pkg/logger"
39-
"github.com/openservicemesh/osm/pkg/messaging"
4042
"github.com/openservicemesh/osm/pkg/metricsstore"
4143
"github.com/openservicemesh/osm/pkg/reconciler"
4244
"github.com/openservicemesh/osm/pkg/signals"
@@ -47,6 +49,8 @@ const (
4749
meshConfigName = "osm-mesh-config"
4850
presetMeshConfigName = "preset-mesh-config"
4951
presetMeshConfigJSONKey = "preset-mesh-config.json"
52+
webhookHealthPath = "/healthz"
53+
healthPort = 9095
5054
)
5155

5256
var (
@@ -57,11 +61,8 @@ var (
5761
meshName string
5862
osmVersion string
5963

60-
crdConverterConfig crdconversion.Config
61-
6264
certProviderKind string
6365

64-
tresorOptions providers.TresorOptions
6566
vaultOptions providers.VaultOptions
6667
certManagerOptions providers.CertManagerOptions
6768

@@ -147,6 +148,8 @@ func main() {
147148
namespace: osmNamespace,
148149
}
149150

151+
applyOrUpdateCRDs(crdClient)
152+
150153
err = bootstrap.ensureMeshConfig()
151154
if err != nil {
152155
log.Fatal().Err(err).Msgf("Error setting up default MeshConfig %s from ConfigMap %s", meshConfigName, presetMeshConfigName)
@@ -158,37 +161,27 @@ func main() {
158161
log.Fatal().Err(err).Msg("Error initializing Kubernetes events recorder")
159162
}
160163

161-
stop := signals.RegisterExitHandlers()
162164
_, cancel := context.WithCancel(context.Background())
163165
defer cancel()
166+
stop := signals.RegisterExitHandlers()
164167

165168
// Start the default metrics store
166169
metricsstore.DefaultMetricsStore.Start(
167170
metricsstore.DefaultMetricsStore.ErrCodeCounter,
168171
metricsstore.DefaultMetricsStore.HTTPResponseTotal,
169172
metricsstore.DefaultMetricsStore.HTTPResponseDuration,
170-
metricsstore.DefaultMetricsStore.ConversionWebhookResourceTotal,
171173
)
172174

173-
msgBroker := messaging.NewBroker(stop)
174-
175-
// Initialize Configurator to retrieve mesh specific config
176-
cfg := configurator.NewConfigurator(configClient, stop, osmNamespace, osmMeshConfigName, msgBroker)
177-
178-
// Intitialize certificate manager/provider
179-
certProviderConfig := providers.NewCertificateProviderConfig(kubeClient, kubeConfig, cfg, providers.Kind(certProviderKind), osmNamespace,
180-
caBundleSecretName, tresorOptions, vaultOptions, certManagerOptions, msgBroker)
181-
182-
certManager, _, err := certProviderConfig.GetCertificateManager()
183-
if err != nil {
184-
events.GenericEventRecorder().FatalEvent(err, events.InvalidCertificateManager,
185-
"Error initializing certificate manager of kind %s", certProviderKind)
186-
}
187-
188-
// Initialize the crd conversion webhook server to support the conversion of OSM's CRDs
189-
crdConverterConfig.ListenPort = constants.CRDConversionWebhookPort
190-
if err := crdconversion.NewConversionWebhook(crdConverterConfig, kubeClient, crdClient, certManager, osmNamespace, enableReconciler, stop); err != nil {
191-
events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating crd conversion webhook")
175+
/*
176+
* Initialize osm-bootstrap's HTTP server
177+
*/
178+
if enableReconciler {
179+
log.Info().Msgf("OSM reconciler enabled for custom resource definitions")
180+
err = reconciler.NewReconcilerClient(kubeClient, apiServerClient, meshName, osmVersion, stop, reconciler.CrdInformerKey)
181+
if err != nil {
182+
events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating reconciler client for custom resource definitions")
183+
log.Fatal().Err(err).Msgf("Failed to create reconcile client for custom resource definitions")
184+
}
192185
}
193186

194187
/*
@@ -199,22 +192,79 @@ func main() {
199192
httpServer.AddHandler(httpserverconstants.MetricsPath, metricsstore.DefaultMetricsStore.Handler())
200193
// Version
201194
httpServer.AddHandler(httpserverconstants.VersionPath, version.GetVersionHandler())
202-
// Start HTTP server
195+
// Webhook
196+
httpServer.AddHandler(webhookHealthPath, metricsstore.AddHTTPMetrics(http.HandlerFunc(healthHandler)))
197+
203198
err = httpServer.Start()
204199
if err != nil {
205200
log.Fatal().Err(err).Msgf("Failed to start OSM metrics/probes HTTP server")
206201
}
207202

208-
if enableReconciler {
209-
log.Info().Msgf("OSM reconciler enabled for custom resource definitions")
210-
err = reconciler.NewReconcilerClient(kubeClient, apiServerClient, meshName, osmVersion, stop, reconciler.CrdInformerKey)
203+
<-stop
204+
cancel()
205+
log.Info().Msgf("Stopping osm-bootstrap %s; %s; %s", version.Version, version.GitCommit, version.BuildDate)
206+
}
207+
208+
func healthHandler(w http.ResponseWriter, _ *http.Request) {
209+
w.WriteHeader(http.StatusOK)
210+
if _, err := w.Write([]byte("Health OK")); err != nil {
211+
log.Error().Err(err).Msg("Error writing bytes for webhook health check handler")
212+
}
213+
}
214+
215+
func applyOrUpdateCRDs(crdClient *apiclient.ApiextensionsV1Client) {
216+
crdFiles, err := filepath.Glob("/osm-crds/*.yaml")
217+
218+
if err != nil {
219+
log.Fatal().Err(err).Msgf("error reading files from /osm-crds")
220+
}
221+
222+
scheme = runtime.NewScheme()
223+
codecs := serializer.NewCodecFactory(scheme)
224+
decode := codecs.UniversalDeserializer().Decode
225+
226+
for _, file := range crdFiles {
227+
yaml, err := os.ReadFile(filepath.Clean(file))
211228
if err != nil {
212-
events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating reconciler client for custom resource definitions")
229+
log.Fatal().Err(err).Msgf("Error reading CRD file %s", file)
213230
}
214-
}
215231

216-
<-stop
217-
log.Info().Msgf("Stopping osm-bootstrap %s; %s; %s", version.Version, version.GitCommit, version.BuildDate)
232+
crd := &apiv1.CustomResourceDefinition{}
233+
_, _, err = decode(yaml, nil, crd)
234+
if err != nil {
235+
log.Fatal().Err(err).Msgf("Error decoding CRD file %s", file)
236+
}
237+
238+
crd.Labels[constants.ReconcileLabel] = strconv.FormatBool(enableReconciler)
239+
240+
crdExisting, err := crdClient.CustomResourceDefinitions().Get(context.Background(), crd.Name, metav1.GetOptions{})
241+
if err != nil && !apierrors.IsNotFound(err) {
242+
log.Fatal().Err(err).Msgf("error getting CRD %s", crd.Name)
243+
}
244+
245+
if apierrors.IsNotFound(err) {
246+
log.Info().Msgf("crds %s not found, creating CRD", crd.Name)
247+
if err := util.CreateApplyAnnotation(crd, unstructured.UnstructuredJSONScheme); err != nil {
248+
log.Fatal().Err(err).Msgf("Error applying annotation to CRD %s", crd.Name)
249+
}
250+
if _, err = crdClient.CustomResourceDefinitions().Create(context.Background(), crd, metav1.CreateOptions{}); err != nil {
251+
log.Fatal().Err(err).Msgf("Error creating crd : %s", crd.Name)
252+
}
253+
log.Info().Msgf("Successfully created crd: %s", crd.Name)
254+
} else {
255+
log.Info().Msgf("Patching conversion webhook configuration for crd: %s, setting to \"None\"", crd.Name)
256+
257+
crdExisting.Labels[constants.ReconcileLabel] = strconv.FormatBool(enableReconciler)
258+
crdExisting.Spec = crd.Spec
259+
crdExisting.Spec.Conversion = &apiv1.CustomResourceConversion{
260+
Strategy: apiv1.NoneConverter,
261+
}
262+
if _, err = crdClient.CustomResourceDefinitions().Update(context.Background(), crdExisting, metav1.UpdateOptions{}); err != nil {
263+
log.Fatal().Err(err).Msgf("Error updating conversion webhook configuration for crd : %s", crd.Name)
264+
}
265+
log.Info().Msgf("successfully set conversion webhook configuration for crd : %s to \"None\"", crd.Name)
266+
}
267+
}
218268
}
219269

220270
func (b *bootstrap) createDefaultMeshConfig() error {

‎dockerfiles/Dockerfile.osm-bootstrap

+1
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
1212

1313
FROM gcr.io/distroless/static
1414
COPY --from=builder /osm/osm-bootstrap /
15+
COPY ./cmd/osm-bootstrap/crds /osm-crds/

‎go.mod

-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ require (
3030
github.com/mholt/archiver/v3 v3.5.0
3131
github.com/mitchellh/gox v1.0.1
3232
github.com/mitchellh/hashstructure/v2 v2.0.1
33-
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
3433
github.com/norwoodj/helm-docs v1.4.0
3534
github.com/olekukonko/tablewriter v0.0.5
3635
github.com/onsi/ginkgo v1.16.5

‎go.sum

-1
Original file line numberDiff line numberDiff line change
@@ -1617,7 +1617,6 @@ github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:
16171617
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
16181618
github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0=
16191619
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
1620-
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
16211620
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
16221621
github.com/mwielbut/pointy v1.1.0/go.mod h1:MvvO+uMFj9T5DMda33HlvogsFBX7pWWKAkFIn4teYwY=
16231622
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

‎pkg/constants/constants.go

-3
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,6 @@ const (
147147

148148
// OSMMeshConfig is the name of the OSM MeshConfig
149149
OSMMeshConfig = "osm-mesh-config"
150-
151-
// CRDConversionWebhookPort is the port of the CRD conversion webhook service
152-
CRDConversionWebhookPort = 9443
153150
)
154151

155152
// Annotations used by the control plane

0 commit comments

Comments
 (0)