Move Flink SQL Gateway to K8s

Author: ryannedolan

Makefile

@@ -45,15 +45,14 @@ undeploy-samples: undeploy
 	kubectl delete -f ./deploy/samples || echo "skipping"
 
 deploy-flink: deploy
+	kubectl create namespace flink || echo "skipping"
 	kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml || echo "skipping"
 	helm repo add flink-operator-repo https://downloads.apache.org/flink/flink-kubernetes-operator-1.9.0/
-	helm upgrade --install --atomic --set webhook.create=false,image.pullPolicy=Never,image.repository=docker.io/library/hoptimator-flink-operator,image.tag=latest flink-kubernetes-operator flink-operator-repo/flink-kubernetes-operator
-	kubectl apply -f deploy/samples/flinkDeployment.yaml
-	kubectl apply -f deploy/samples/flinkSessionJob.yaml
-	docker compose -f ./deploy/docker/flink/docker-compose-sql-gateway.yaml up -d --wait
+	helm upgrade --install --atomic --set webhook.create=false,image.pullPolicy=Never,image.repository=docker.io/library/hoptimator-flink-operator,image.tag=latest --set-json='watchNamespaces=["default","flink"]' flink-kubernetes-operator flink-operator-repo/flink-kubernetes-operator
+	kubectl apply -f deploy/dev/flink-session-cluster.yaml
+	kubectl apply -f deploy/samples/flink-template.yaml
 
 undeploy-flink:
-	docker compose -f ./deploy/docker/flink/docker-compose-sql-gateway.yaml down
 	kubectl delete flinksessionjobs.flink.apache.org --all || echo "skipping"
 	kubectl delete flinkdeployments.flink.apache.org --all || echo "skipping"
 	kubectl delete crd flinksessionjobs.flink.apache.org || echo "skipping"
@@ -106,15 +105,19 @@ integration-tests: deploy-dev-environment
 	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
 	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
 	kubectl port-forward -n kafka svc/one-kafka-external-bootstrap 9092 & echo $$! > port-forward.pid
-	./gradlew intTest || kill `cat port-forward.pid`
+	kubectl port-forward -n flink svc/flink-sql-gateway 8083 & echo $$! > port-forward-2.pid
+	kubectl port-forward -n flink svc/basic-session-deployment-rest 8081 & echo $$! > port-forward-3.pid
+	./gradlew intTest || kill `cat port-forward.pid port-forward-2.pid, port-forward-3.pid`
 	kill `cat port-forward.pid`
+	kill `cat port-forward-2.pid`
+	kill `cat port-forward-3.pid`
 
 # kind cluster used in github workflow needs to have different routing set up, avoiding the need to forward kafka ports
 integration-tests-kind: deploy-dev-environment
 	kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
 	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
 	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
-	./gradlew intTest
+	./gradlew intTest -i
 
 generate-models:
 	./generate-models.sh

deploy/samples/flinkDeployment.yaml renamed to deploy/dev/flink-session-cluster.yaml

@@ -2,6 +2,7 @@ apiVersion: flink.apache.org/v1beta1
 kind: FlinkDeployment
 metadata:
   name: basic-session-deployment
+  namespace: flink
 spec:
   image: docker.io/library/hoptimator-flink-runner
   imagePullPolicy: Never

deploy/dev/flink-sql-gateway.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flink-sql-gateway
+  namespace: flink
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flink-sql-gateway
+  template:
+    metadata:
+      labels:
+        app: flink-sql-gateway
+    spec:
+      containers:
+      - name: flink-sql-gateway
+        image: flink:1.18.1
+        ports:
+        - containerPort: 8083
+        command: ["./bin/sql-gateway.sh"]
+        args:
+        - "start-foreground"
+        - "-Dsql-gateway.endpoint.rest.address=localhost"
+        - "-Drest.address=basic-session-deployment-rest.flink.svc.cluster.local"
+
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-sql-gateway
+  namespace: flink
+spec:
+  type: NodePort
+  ports:
+  - name: rest
+    port: 8083
+    protocol: TCP
+    targetPort: 8083
+    nodePort: 30083
+  selector:
+    app: flink-sql-gateway
+
+

deploy/docker/flink/docker-compose-sql-gateway.yaml

@@ -10,6 +10,7 @@ spec:
     kind: FlinkSessionJob
     metadata:
       name: {{name}}
+      namespace: flink
     spec:
       deploymentName: basic-session-deployment
       job:

deploy/samples/flinkengine.yaml renamed to deploy/samples/flink-engine.yaml

@@ -15,4 +15,12 @@ nodes:
     - containerPort: 31234
       hostPort: 31234
       listenAddress: "127.0.0.1"
-      protocol: TCP
+      protocol: TCP
+    - containerPort: 30083
+      hostPort: 8083
+      listenAddress: "127.0.0.1"
+      protocol: TCP
+    - containerPort: 30081
+      hostPort: 8081
+      listenAddress: "127.0.0.1"
+      protocol: TCP

deploy/samples/flinkSessionJob.yaml renamed to deploy/samples/flink-template.yaml

@@ -6,6 +6,7 @@ apiVersion: flink.apache.org/v1beta1
 kind: FlinkSessionJob
 metadata:
   name: kafka-database-existing-topic-1
+  namespace: flink
 spec:
   deploymentName: basic-session-deployment
   job:

etc/cluster.yaml

@@ -6,6 +6,7 @@ apiVersion: flink.apache.org/v1beta1
 kind: FlinkSessionJob
 metadata:
   name: venice-cluster0-test-store-1
+  namespace: flink
 spec:
   deploymentName: basic-session-deployment
   job:

hoptimator-kafka/src/test/resources/kafka-ddl.id

@@ -6,6 +6,7 @@ apiVersion: flink.apache.org/v1beta1
 kind: FlinkSessionJob
 metadata:
   name: venice-cluster0-test-store-1
+  namespace: flink
 spec:
   deploymentName: basic-session-deployment
   job:

hoptimator-venice/src/test/resources/venice-ddl-insert-all.id

@@ -6,6 +6,7 @@ apiVersion: flink.apache.org/v1beta1
 kind: FlinkSessionJob
 metadata:
   name: pipeline-sink
+  namespace: flink
 spec:
   deploymentName: basic-session-deployment
   job: