HTTPSO-based Routing Table (#669)

Author: t0rr3sp3dr0

.devcontainer/Dockerfile

@@ -54,14 +54,6 @@ RUN apt-get update \
         github.com/derekparker/delve/cmd/dlv 2>&1 \
     && go install honnef.co/go/tools/cmd/staticcheck@latest \
     && go install golang.org/x/tools/gopls@latest \
-    # Protocol Buffer Compiler
-    && PROTOC_VERSION=21.9 \
-    && if [ $(dpkg --print-architecture) = "amd64" ]; then PROTOC_ARCH="x86_64"; else PROTOC_ARCH="aarch_64" ; fi \
-    && curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-$PROTOC_ARCH.zip" \
-    && unzip "protoc-${PROTOC_VERSION}-linux-$PROTOC_ARCH.zip" -d $HOME/.local \
-    && mv $HOME/.local/bin/protoc /usr/local/bin/protoc \
-    && mv $HOME/.local/include/ /usr/local/bin/include/ \
-    && protoc --version \
     # Install golangci-lint
     && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0 \
     #

.dockerignore

@@ -1,16 +1,14 @@
-/target
+*.md
+*.yaml
+*.yml
+.*
+/LICENSE
 /bin
-.git
-.gitignore
 /charts
-/.vscode
-/bin
 /cli
-/examples
+/config
 /docs
-/.envrc
-/.github
-/README.md
-/RELEASE_PROCESS.md
-CONTRIBUTING.md
+/examples
+/target
+/tests
 Dockerfile

.github/workflows/tests.yaml

@@ -53,6 +53,9 @@ jobs:
     - name: Manifests
       run: make verify-manifests
 
+    - name: Mockgen
+      run: make verify-mockgen
+
     - name: Build
       run: ARCH=${{ matrix.name }} make build
 
@@ -71,5 +74,5 @@ jobs:
         with:
           go-version: 1.19
       - name: Get golangci
-        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2
       - uses: pre-commit/[email protected]

.golangci.yml

@@ -52,6 +52,14 @@ issues:
         - gomnd
         - dupl
         - unparam
+  # Exclude gci check for //+kubebuilder:scaffold:imports comments. Waiting to
+  # resolve https://github.com/kedacore/keda/issues/4379
+    - path: operator/controllers/http/suite_test.go
+      linters:
+        - gci
+    - path: operator/main.go
+      linters:
+        - gci
 linters-settings:
   funlen:
     lines: 80

CHANGELOG.md

@@ -26,6 +26,7 @@ This changelog keeps track of work items that have been completed and are ready
 - **General**: Automatically tag Docker image with commit SHA ([#567](https://github.com/kedacore/http-add-on/issues/567))
 - **RBAC**: Introduce fine-grained permissions per component and reduce required permissions ([#612](https://github.com/kedacore/http-add-on/issues/612))
 - **Operator**: Migrate project to Kubebuilder v3 ([#625](https://github.com/kedacore/http-add-on/issues/625))
+- **Routing**: New routing table implementation that relies on the live state of HTTPScaledObjects on the K8s Cluster instead of a ConfigMap that is updated periodically ([#605](https://github.com/kedacore/http-add-on/issues/605))
 
 ### Fixes
 

Makefile

@@ -34,14 +34,14 @@ GIT_COMMIT_SHORT  ?= $(shell git rev-parse --short HEAD)
 
 # Build targets
 
-build-operator: proto-gen
-	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -a -o bin/operator ./operator
+build-operator:
+	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -trimpath -a -o bin/operator ./operator
 
-build-interceptor: proto-gen
-	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -a -o bin/interceptor ./interceptor
+build-interceptor:
+	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -trimpath -a -o bin/interceptor ./interceptor
 
-build-scaler: proto-gen
-	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -a -o bin/scaler ./scaler
+build-scaler:
+	${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -trimpath -a -o bin/scaler ./scaler
 
 build: build-operator build-interceptor build-scaler
 
@@ -85,9 +85,9 @@ publish-multiarch: publish-operator-multiarch publish-interceptor-multiarch publ
 
 # Development
 
-generate: codegen manifests ## Generate code and manifests.
+generate: codegen manifests mockgen ## Generate code, manifests, and mocks.
 
-verify: verify-codegen verify-manifests ## Verify code and manifests.
+verify: verify-codegen verify-manifests verify-mockgen ## Verify code, manifests, and mocks.
 
 codegen: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile='hack/boilerplate.go.txt' paths='./...'
@@ -104,18 +104,24 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 verify-manifests: ## Verify manifests are up to date.
 	./hack/verify-manifests.sh
 
+mockgen: ## Generate mock implementations of Go interfaces.
+	./hack/update-mockgen.sh
+
+verify-mockgen: ## Verify mocks are up to date.
+	./hack/verify-mockgen.sh
+
 fmt: ## Run go fmt against code.
 	go fmt ./...
 
 vet: ## Run go vet against code.
 	go vet ./...
 
+lint: ## Run golangci-lint against code.
+	golangci-lint run
+
 pre-commit: ## Run static-checks.
 	pre-commit run --all-files
 
-proto-gen: protoc-gen-go ## Scaler protobuffers
-	protoc --proto_path=proto scaler.proto --go_out=proto --go-grpc_out=proto
-
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
 	GOBIN=$(shell pwd)/bin go install sigs.k8s.io/controller-tools/cmd/[email protected]
@@ -124,10 +130,6 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
 	GOBIN=$(shell pwd)/bin go install sigs.k8s.io/kustomize/kustomize/[email protected]
 
-protoc-gen-go: ## Download protoc-gen-go
-	go install google.golang.org/protobuf/cmd/[email protected]
-	go install google.golang.org/grpc/cmd/[email protected]
-
 deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/config.
 	cd config/interceptor && \
 	$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-interceptor=${IMAGE_INTERCEPTOR_VERSIONED_TAG}
@@ -142,3 +144,8 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
 
 undeploy:
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+kind-load:
+	kind load docker-image ghcr.io/kedacore/http-add-on-operator:${VERSION}
+	kind load docker-image ghcr.io/kedacore/http-add-on-interceptor:${VERSION}
+	kind load docker-image ghcr.io/kedacore/http-add-on-scaler:${VERSION}

config/crd/bases/http.keda.sh_httpscaledobjects.yaml

@@ -88,7 +88,7 @@ spec:
                 type: object
               scaleTargetRef:
                 description: The name of the deployment to route HTTP requests to
-                  (and to autoscale). Either this or Image must be set
+                  (and to autoscale).
                 properties:
                   deployment:
                     description: The name of the deployment to scale according to

config/interceptor/admin.service.yaml

@@ -1,4 +1,3 @@
-# TODO(pedrotorres): remove after implementing new routing table
 apiVersion: v1
 kind: Service
 metadata:

config/interceptor/deployment.yaml

@@ -53,11 +53,18 @@ spec:
         - name: KEDA_HTTP_EXPECT_CONTINUE_TIMEOUT
           value: "1s"
         ports:
-        # TODO(pedrotorres): remove after implementing new routing table
         - name: admin
           containerPort: 9090
         - name: proxy
           containerPort: 8080
+        livenessProbe:
+          httpGet:
+            path: /livez
+            port: proxy
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: proxy
         # TODO(pedrotorres): set better default values avoiding overcommitment
         resources:
           requests:
@@ -69,11 +76,12 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsNonRoot: true
           capabilities:
             drop:
-            - ALL
-          seccompProfile:
-            type: RuntimeDefault
+              - ALL
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: interceptor
       terminationGracePeriodSeconds: 10

config/interceptor/role.yaml

@@ -12,17 +12,10 @@ rules:
   - get
   - list
   - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: interceptor
-  namespace: keda
-rules:
 - apiGroups:
-  - ""
+  - http.keda.sh
   resources:
-  - configmaps
+  - httpscaledobjects
   verbs:
   - get
   - list

config/interceptor/role_binding.yaml

@@ -10,15 +10,3 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: interceptor
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: interceptor
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: interceptor
-subjects:
-- kind: ServiceAccount
-  name: interceptor

config/operator/deployment.yaml

@@ -27,25 +27,28 @@ spec:
         # TODO(pedrotorres): remove after implementing new routing table
         - --admin-port=9090
         env:
-        # TODO(pedrotorres): remove after implementing new routing table
-        - name: KEDAHTTP_INTERCEPTOR_SERVICE
-          value: "keda-http-add-on-interceptor-admin"
         - name: KEDAHTTP_OPERATOR_EXTERNAL_SCALER_SERVICE
           value: "keda-http-add-on-external-scaler"
         - name: KEDAHTTP_OPERATOR_EXTERNAL_SCALER_PORT
           value: "9090"
-        - name: KEDAHTTP_INTERCEPTOR_ADMIN_PORT
-          value: "9090"
-        - name: KEDAHTTP_INTERCEPTOR_PROXY_PORT
-          value: "8080"
         - name: KEDA_HTTP_OPERATOR_NAMESPACE
           value: "keda"
         - name: KEDA_HTTP_OPERATOR_WATCH_NAMESPACE
           value: ""
         # TODO(pedrotorres): remove after implementing new routing table
         ports:
-        - name: admin
-          containerPort: 9090
+        - name: metrics
+          containerPort: 8080
+        - name: probes
+          containerPort: 8081
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: probes
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: probes
         # TODO(pedrotorres): set better default values avoiding overcommitment
         resources:
           requests:
@@ -57,11 +60,12 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsNonRoot: true
           capabilities:
             drop:
-            - ALL
-          seccompProfile:
-            type: RuntimeDefault
+              - ALL
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: operator
       terminationGracePeriodSeconds: 10

config/scaler/deployment.yaml

@@ -23,13 +23,10 @@ spec:
       - name: scaler
         image: ghcr.io/kedacore/http-add-on-scaler
         env:
-        # TODO(pedrotorres): remove after implementing new routing table
         - name: KEDA_HTTP_SCALER_TARGET_ADMIN_DEPLOYMENT
           value: "keda-http-add-on-interceptor"
         - name: KEDA_HTTP_SCALER_PORT
           value: "9090"
-        - name: KEDA_HTTP_HEALTH_PORT
-          value: "9091"
         - name: KEDA_HTTP_SCALER_TARGET_ADMIN_NAMESPACE
           value: "keda"
           # TODO(pedrotorres): remove after implementing new routing table
@@ -43,8 +40,14 @@ spec:
         ports:
         - name: grpc
           containerPort: 9090
-        - name: health
-          containerPort: 9091
+        livenessProbe:
+          grpc:
+            port: 9090
+            service: liveness
+        readinessProbe:
+          grpc:
+            port: 9090
+            service: readiness
         # TODO(pedrotorres): set better default values avoiding overcommitment
         resources:
           requests:
@@ -56,11 +59,12 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsNonRoot: true
           capabilities:
             drop:
             - ALL
-          seccompProfile:
-            type: RuntimeDefault
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: scaler
       terminationGracePeriodSeconds: 10

config/scaler/role.yaml

@@ -20,6 +20,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - http.keda.sh
+  resources:
+  - httpscaledobjects
+  verbs:
+  - get
+  - list
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

config/scaler/service.yaml

@@ -8,7 +8,3 @@ spec:
     protocol: TCP
     port: 9090
     targetPort: grpc
-  - name: health
-    protocol: TCP
-    port: 9091
-    targetPort: health