From 3501da704635bd2b5deaa421359cc5670c6598ed Mon Sep 17 00:00:00 2001 From: Matei Grigore Date: Thu, 18 Jul 2024 12:23:26 +0100 Subject: [PATCH 1/5] Add changes that improve the local setup process --- Makefile | 15 +++++++-------- docs/build_operator_locally.md | 2 +- docs/contributing.md | 21 +++++++++++++++------ docs/run-operator-locally.md | 22 ++++++++++++++-------- inventory.yaml | 4 ++-- pipeline.py | 18 +++++++++++++----- scripts/dev/setup_kind_cluster.sh | 24 ++++++++++++++++++++---- 7 files changed, 72 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index a03dfb9bd..6f1811c8f 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,7 @@ NAMESPACE := $(shell jq -r .namespace < $(MONGODB_COMMUNITY_CONFIG)) UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < $(MONGODB_COMMUNITY_CONFIG)) READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < $(MONGODB_COMMUNITY_CONFIG)) REGISTRY := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) -AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubi < $(MONGODB_COMMUNITY_CONFIG)) - +AGENT_IMAGE_NAME := $(shell jq -r .agent_image < $(MONGODB_COMMUNITY_CONFIG)) HELM_CHART ?= ./helm-charts/charts/community-operator STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) @@ -134,7 +133,7 @@ install-rbac: $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl apply -f - uninstall-crd: - kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com + kubectl delete crd --ignore-not-found mongodbcommunity.mongodbcommunity.mongodb.com uninstall-chart: $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) @@ -193,19 +192,19 @@ generate-env-file: ## generates a local-test.env for local testing ##@ Image operator-image: ## Build and push the operator image - python pipeline.py --image-name operator + python pipeline.py --image-name operator $(IMG_BUILD_ARGS) e2e-image: ## Build and push e2e test image - python pipeline.py --image-name e2e + python pipeline.py --image-name e2e $(IMG_BUILD_ARGS) agent-image: ## Build and push agent image - python pipeline.py --image-name agent + python pipeline.py --image-name agent $(IMG_BUILD_ARGS) readiness-probe-image: ## Build and push readiness probe image - python pipeline.py --image-name readiness-probe + python pipeline.py --image-name readiness-probe $(IMG_BUILD_ARGS) version-upgrade-post-start-hook-image: ## Build and push version upgrade post start hook image - python pipeline.py --image-name version-upgrade-hook + python pipeline.py --image-name version-upgrade-hook $(IMG_BUILD_ARGS) all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image ## create all required images diff --git a/docs/build_operator_locally.md b/docs/build_operator_locally.md index d1219c654..33dfff340 100644 --- a/docs/build_operator_locally.md +++ b/docs/build_operator_locally.md @@ -40,7 +40,7 @@ git submodule update --init ``` -5. Build and deploy the operator: +5. Build and deploy the operator. Also add `IMG_BUILD_ARGS=--insecure` as described [here](contributing.md#deploying-the-operator) if necessary: ```sh # builds all required images and then deploys the operator diff --git a/docs/contributing.md b/docs/contributing.md index d1414eb86..139d11b71 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -58,9 +58,9 @@ to be able to run properly. Create a json file with the following content: "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "community-operator-e2e", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "version_upgrade_hook_image": "mongodb-kubernetes-operator-version-upgrade-post-start-hook", "agent_image": "mongodb-agent-ubi-dev", - "readiness_probe_image": "mongodb-kubernetes-readiness", + "readiness_probe_image": "mongodb-kubernetes-readinessprobe", "s3_bucket": "" } ``` @@ -132,6 +132,14 @@ make operator-image deploy This will build and deploy the operator to namespace specified in your configuration file. +If you are using a local docker registry you should run the following command. +The additional `IMG_BUILD_ARGS=--insecure` variable will add the `--insecure` flag to the command creating the manifests. +This is necessary if your local registry is not secure. Read more about the flag on the [documentatio](https://docs.docker.com/reference/cli/docker/manifest/#working-with-insecure-registries) + +```sh +IMG_BUILD_ARGS=--insecure make operator-image deploy +``` + #### See the operator deployment ```sh @@ -149,7 +157,7 @@ To remove the operator and any created resources you can run make undeploy ``` -Alternatively, you can run the operator locally with +Alternatively, you can run the operator locally. Make sure you follow the steps outlined in [run-operator-locally.md](run-operator-locally.md) ```sh make run @@ -168,7 +176,8 @@ make test ### E2E Tests If this is the first time running E2E tests, you will need to ensure that you have built and pushed -all images required by the E2E tests. You can do this by running. +all images required by the E2E tests. You can do this by running the following command, +or with the additional `IMG_BUILD_ARGS=--insecure` described above. ```sh make all-images @@ -180,7 +189,7 @@ For subsequent tests you can use make e2e-k8s test= ``` -This will only re-build the e2e test image. +This will only re-build the e2e test image. Add `IMG_BUILD_ARGS=--insecure` if necessary We have built a simple mechanism to run E2E tests on your cluster using a runner that deploys a series of Kubernetes objects, runs them, and awaits for their @@ -199,7 +208,7 @@ replica_set_scale ... ``` -The tests should run individually using the runner like this: +The tests should run individually using the runner like this, or additionally with `IMG_BUILD_ARGS=--insecure`: ```sh make e2e-k8s test=replica_set diff --git a/docs/run-operator-locally.md b/docs/run-operator-locally.md index a46ecba56..f5acb2f26 100644 --- a/docs/run-operator-locally.md +++ b/docs/run-operator-locally.md @@ -18,17 +18,23 @@ Being able to run and build the binary locally can help with faster feedback-cyc - Run e2e tests locally ## Running The Operator locally -1. Use the dedicated make target which exports the needed environment variables and builds & runs the operator binary - -```sh -make run -``` +1. Use the dedicated make target which exports the needed environment variables and builds & runs the operator binary. + + Before doing that you need to add 2 more fields to the `config.json` file from in [contributing.md](contributing.md): + - `mdb_local_operator`: needs to be set to `true` + - `kubeconfig`: needs to be set to the path of the configuration file mentioned above + + Then you can run the command: + + ```sh + make run + ``` 2. For debugging one can use the following make target, which uses [dlv](https://github.com/go-delve/delve): -```sh -make debug -``` + ```sh + make debug + ``` ## Running e2e tests with the local operator - Our [e2e tests](../test/e2e), contains sub-steps that will install the following helm-chart: [operator.yaml](../helm-charts/charts/community-operator/templates/operator.yaml) diff --git a/inventory.yaml b/inventory.yaml index 28d475740..ba9e005b5 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -83,8 +83,8 @@ images: buildargs: agent_version: $(inputs.params.release_version) tools_version: $(inputs.params.tools_version) - agent_distro: rhel7_x86_64 - tools_distro: rhel70-x86_64 + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) labels: quay.expires-after: Never diff --git a/pipeline.py b/pipeline.py index 078a096eb..300792937 100644 --- a/pipeline.py +++ b/pipeline.py @@ -87,6 +87,7 @@ def build_and_push_image( architectures: Set[str], release: bool, sign: bool, + insecure: bool, ) -> None: if sign: mongodb_artifactory_login() @@ -119,16 +120,18 @@ def build_and_push_image( else: raise Exception("Dev image must be specified") - push_manifest(config, architectures, image_to_push) + push_manifest(config, architectures, image_to_push, insecure) if config.gh_run_id: - push_manifest(config, architectures, image_to_push, config.gh_run_id) + push_manifest(config, architectures, image_to_push, insecure, config.gh_run_id) if release: registry = args["registry"] + "/" + args["image"] context_tag = args["release_version"] + "-context" - push_manifest(config, architectures, args["image"], args["release_version"]) - push_manifest(config, architectures, args["image"], context_tag) + push_manifest( + config, architectures, args["image"], insecure, args["release_version"] + ) + push_manifest(config, architectures, args["image"], insecure, context_tag) if sign: sign_and_verify(registry, args["release_version"]) sign_and_verify(registry, context_tag) @@ -149,6 +152,7 @@ def push_manifest( config: DevConfig, architectures: Set[str], image_name: str, + insecure: bool = False, image_tag: str = "latest", ) -> None: logger.info(f"Pushing manifest for {image_tag}") @@ -164,6 +168,9 @@ def push_manifest( final_manifest, ] + if insecure: + create_args.append("--insecure") + for arch in architectures: create_args.extend(["--amend", final_manifest + "-" + arch]) @@ -220,6 +227,7 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument("--tag", type=str) parser.add_argument("--sign", action="store_true", default=False) + parser.add_argument("--insecure", action="store_true", default=False) return parser.parse_args() @@ -277,7 +285,7 @@ def main() -> int: image_args = build_image_args(config, image_name) build_and_push_image( - image_name, config, image_args, arch_set, args.release, args.sign + image_name, config, image_args, arch_set, args.release, args.sign, args.insecure ) return 0 diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 7f19c0d6e..3178f2878 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -56,7 +56,7 @@ reg_name='kind-registry' reg_port='5000' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" if [ "${running}" != 'true' ]; then - docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" registry:2 + docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network kind --name "${reg_name}" registry:2 fi if [ "${recreate}" != 0 ]; then @@ -72,10 +72,26 @@ networking: serviceSubnet: "${service_network}" containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"] - endpoint = ["http://${reg_name}:${reg_port}"] + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" EOF +# Add the registry config to the nodes +# +# This is necessary because localhost resolves to loopback addresses that are +# network-namespace local. +# In other words: localhost in the container is not localhost on the host. +# +# We want a consistent name that works from both ends, so we tell containerd to +# alias localhost:${reg_port} to the registry container when pulling images +REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}" +for node in $(kind get nodes --name "${cluster_name}"); do + docker exec "${node}" mkdir -p "${REGISTRY_DIR}" + cat < Date: Thu, 18 Jul 2024 16:03:38 +0100 Subject: [PATCH 2/5] Remove default values for agent_distro and tools_distro from inventory.yaml --- inventory.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/inventory.yaml b/inventory.yaml index ba9e005b5..e2a37214c 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -9,9 +9,6 @@ images: vars: context: . template_context: scripts/dev/templates/agent - # Default values but overwritten in pipeline.py - agent_distro: rhel7_x86_64 - tools_distro: rhel70-x86_64 inputs: - release_version From efe8a34b2366a0634fc691c3e9c3a2e3870f3502 Mon Sep 17 00:00:00 2001 From: Matei Grigore Date: Thu, 18 Jul 2024 17:36:25 +0100 Subject: [PATCH 3/5] Add suggested changes --- pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.py b/pipeline.py index 300792937..b8324d249 100644 --- a/pipeline.py +++ b/pipeline.py @@ -87,7 +87,7 @@ def build_and_push_image( architectures: Set[str], release: bool, sign: bool, - insecure: bool, + insecure: bool = False, ) -> None: if sign: mongodb_artifactory_login() From 335151dad7d6af8c73a73113583f694e75a25002 Mon Sep 17 00:00:00 2001 From: Matei Grigore Date: Mon, 22 Jul 2024 14:30:08 +0100 Subject: [PATCH 4/5] Make explanation in the documentation more clear --- docs/run-operator-locally.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/run-operator-locally.md b/docs/run-operator-locally.md index f5acb2f26..c742f54b7 100644 --- a/docs/run-operator-locally.md +++ b/docs/run-operator-locally.md @@ -20,9 +20,9 @@ Being able to run and build the binary locally can help with faster feedback-cyc ## Running The Operator locally 1. Use the dedicated make target which exports the needed environment variables and builds & runs the operator binary. - Before doing that you need to add 2 more fields to the `config.json` file from in [contributing.md](contributing.md): - - `mdb_local_operator`: needs to be set to `true` - - `kubeconfig`: needs to be set to the path of the configuration file mentioned above + Before doing that you need to add 2 more fields to the `config.json` file found in [contributing.md](contributing.md), because the python script looks for them in the file: + - `mdb_local_operator`: needs to be set to `true`, to allow for the operator to be run locally + - `kubeconfig`: needs to be set to the path of the `kubeconfig` configuration file, for example `$HOME/.kube/config` Then you can run the command: From f8cad3018c4de2cb9e42e7652ffdd6072d05f91a Mon Sep 17 00:00:00 2001 From: Matei Grigore Date: Tue, 23 Jul 2024 15:59:57 +0100 Subject: [PATCH 5/5] Remove additional changes --- scripts/dev/setup_kind_cluster.sh | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 3178f2878..eab968b22 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -56,7 +56,7 @@ reg_name='kind-registry' reg_port='5000' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" if [ "${running}" != 'true' ]; then - docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network kind --name "${reg_name}" registry:2 + docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" registry:2 fi if [ "${recreate}" != 0 ]; then @@ -72,26 +72,10 @@ networking: serviceSubnet: "${service_network}" containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"] + endpoint = ["http://${reg_name}:${reg_port}"] EOF -# Add the registry config to the nodes -# -# This is necessary because localhost resolves to loopback addresses that are -# network-namespace local. -# In other words: localhost in the container is not localhost on the host. -# -# We want a consistent name that works from both ends, so we tell containerd to -# alias localhost:${reg_port} to the registry container when pulling images -REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}" -for node in $(kind get nodes --name "${cluster_name}"); do - docker exec "${node}" mkdir -p "${REGISTRY_DIR}" - cat <