Skip to content

Commit 33b4479

Browse files
pstpst
committed
Add Kubestack integration test
1 parent 6f619d5 commit 33b4479

18 files changed

+1033
-1
lines changed

‎.github/workflows/main.yml

+68-1
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,76 @@ jobs:
218218
- name: 'Terraform Apply'
219219
run: terraform apply --auto-approve
220220

221+
int_test_kubestack_kind:
222+
runs-on: ubuntu-latest
223+
needs: [compile_provider]
224+
225+
steps:
226+
- name: 'Checkout'
227+
uses: actions/checkout@v1
228+
229+
- name: 'Download terraform-plugins'
230+
uses: actions/download-artifact@v2
231+
with:
232+
name: terraform-plugins
233+
path: tests/kubestack-starter-kind/terraform.d/plugins
234+
235+
- name: 'Ensure provider is executable'
236+
run: chmod +x tests/kubestack-starter-kind/terraform.d/plugins/registry.terraform.io/kbst/kustomization/1.0.0/linux_amd64/terraform-provider-kustomization_v1.0.0
237+
238+
- name: 'Build Kubestack Image'
239+
run: |
240+
docker build \
241+
-t test-image:${{ github.sha }} \
242+
tests/kubestack-starter-kind/
243+
244+
- name: 'Terraform Init'
245+
run: |
246+
docker run \
247+
--rm \
248+
--privileged \
249+
--net host \
250+
-v `pwd`/tests/kubestack-starter-kind:/infra \
251+
-v /var/run/docker.sock:/var/run/docker.sock \
252+
test-image:${{ github.sha }} \
253+
terraform init
254+
255+
- name: 'Terraform Workspace'
256+
run: |
257+
docker run \
258+
--rm \
259+
--privileged \
260+
--net host \
261+
-v `pwd`/tests/kubestack-starter-kind:/infra \
262+
-v /var/run/docker.sock:/var/run/docker.sock \
263+
test-image:${{ github.sha }} \
264+
terraform workspace new apps
265+
266+
- name: 'Terraform Apply'
267+
run: |
268+
docker run \
269+
--rm \
270+
--privileged \
271+
--net host \
272+
-v `pwd`/tests/kubestack-starter-kind:/infra \
273+
-v /var/run/docker.sock:/var/run/docker.sock \
274+
test-image:${{ github.sha }} \
275+
terraform apply --auto-approve
276+
277+
- name: 'Terraform Destroy'
278+
run: |
279+
docker run \
280+
--rm \
281+
--privileged \
282+
--net host \
283+
-v `pwd`/tests/kubestack-starter-kind:/infra \
284+
-v /var/run/docker.sock:/var/run/docker.sock \
285+
test-image:${{ github.sha }} \
286+
terraform apply --auto-approve
287+
221288
goreleaser:
222289
runs-on: ubuntu-latest
223-
needs: [tf_tests, int_test_kubeconfig_path, int_test_kubeconfig_raw, int_test_state_import]
290+
needs: [tf_tests, int_test_kubeconfig_path, int_test_kubeconfig_raw, int_test_state_import, int_test_kubestack_kind]
224291
if: startsWith(github.ref, 'refs/tags/v')
225292
steps:
226293
- name: Checkout

‎tests/kubestack-starter-kind/.gitignore

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Local .terraform directories
2+
**/.terraform/*
3+
4+
# .tfstate files
5+
*.tfstate
6+
*.tfstate.*
7+
8+
# .user home directory
9+
.user/
10+
11+
# terraform generated clusters directory
12+
clusters/

‎tests/kubestack-starter-kind/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
FROM kubestack/framework:v0.13.0-beta.0-kind

‎tests/kubestack-starter-kind/README.md

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Welcome to Kubestack
2+
3+
This repository uses [Kubestack][1]. Kubestack is the open source GitOps framework for teams that want to automate infrastructure, not reinvent automation.
4+
5+
* Infrastructure is defined using Terraform configuration
6+
* Cluster manifests are defined using Kustomize bases and overlays
7+
* Bases and overlays can be bespoke, or consumed from the [catalog][2].
8+
* Both infrastructure and manifests follow the Kubestack [inheritance model][3] to prevent configuration drift between the *ops* and *apps* environments
9+
* All changes follow the same four step process.
10+
11+
Full [framework documentation][4] is available online.
12+
13+
## Making changes
14+
15+
All changes to the Kubernetes cluster, supporting infrastructure and the services defined as part of the manifests in this repository follow the Kubestack [GitOps process][5]. The GitOps process ensures that changes are safely applied by first reviewing the proposed changes, then validating the changes against the *ops* environment and only then promoting the changes to be applied against the *apps* environment by setting a tag.
16+
17+
To accelerate the developer workflow, a [development environment][6], can be run on localhost.
18+
19+
1. Change
20+
21+
Make changes to the configuration in a new branch. Commit the changed configuration. Validate your changes by pushing the new branch. The pipeline runs `terraform plan` against the *ops* workspace.
22+
23+
```shell
24+
# checkout a new branch from master
25+
git checkout -b examplechange master
26+
27+
# make your changes
28+
29+
# commit your changes
30+
git commit # write a meaningful commit message
31+
32+
# push your changes
33+
git push origin examplechange
34+
```
35+
36+
1. Review
37+
38+
Request a peer review of your changes. Team members review the changes and the Terraform plan. If reviewers require changes, make additional commits in the branch.
39+
40+
```shell
41+
# make sure you're in the correct branch
42+
git checkout examplechange
43+
44+
# make changes required by the review
45+
46+
# commit and push the required changes
47+
git commit # write a meaningful commit message
48+
git push origin examplechange
49+
```
50+
51+
1. Merge
52+
53+
If approved, merge your changes to master, to apply them against the *ops* environment. After applying to *ops* was successful, the pipeline runs Terraform plan against the *apps* environment.
54+
55+
```shell
56+
# you can merge on the commandline
57+
# or by merging a pull request
58+
git checkout master
59+
git merge examplechange
60+
git push origin master
61+
```
62+
63+
1. Promote
64+
65+
Review the previous *apps* environment plan and tag the merge commit to promote the same changes to the *apps* environment.
66+
67+
```shell
68+
# make sure you're on the correct commit
69+
git checkout master
70+
git pull
71+
git log -1
72+
73+
# if correct, tag the current commit
74+
# any tag prefixed with `apps-deploy-`
75+
# will trigger the pipeline
76+
git tag apps-deploy-$(date -I)-0
77+
78+
# in case of multiple deploys on the same day,
79+
# increase the counter
80+
# e.g. git tag apps-deploy-2020-05-14-1
81+
```
82+
83+
## Manual operations
84+
85+
In case of the automation being unavailable, upgrades requiring manual steps or in disaster recovery scenarios run Terraform and the cloud CLI locally. Kubestack provides container images bundling all dependencies to use for both automated and manual operations.
86+
87+
1. Exec into container
88+
89+
```shell
90+
# Build the bootstrap container
91+
docker build -t kubestack .
92+
93+
# Exec into the bootstrap container
94+
# add docker socket mount for local dev
95+
# -v /var/run/docker.sock:/var/run/docker.sock
96+
docker run --rm -ti \
97+
-v `pwd`:/infra \
98+
kubestack
99+
```
100+
101+
1. Authenticate providers
102+
103+
Credentials are cached inside the `.user` directory. The directory is excluded from Git by the default `.gitignore`.
104+
105+
```shell
106+
# for AWS
107+
aws configure
108+
109+
# for Azure
110+
az login
111+
112+
# for GCP
113+
gcloud init
114+
gcloud auth application-default login
115+
```
116+
117+
1. Select desired environment
118+
119+
```shell
120+
# for ops
121+
terraform workspace select ops
122+
123+
# or for apps
124+
terraform workspace select apps
125+
```
126+
127+
1. Run Terraform commands
128+
129+
```shell
130+
# run terraform init
131+
terraform init
132+
133+
# run, e.g. terraform plan
134+
terraform plan
135+
```
136+
137+
[1]: https://www.kubestack.com
138+
[2]: https://www.kubestack.com/catalog
139+
[3]: https://www.kubestack.com/framework/documentation/inheritance-model
140+
[4]: https://www.kubestack.com/framework/documentation
141+
[5]: https://www.kubestack.com/framework/documentation/gitops-process
142+
[6]: https://www.kubestack.com/framework/documentation/tutorial-build-local-lab

‎tests/kubestack-starter-kind/clusters.tf

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module "kind_zero" {
2+
source = "github.com/kbst/terraform-kubestack//kind/cluster?ref=v0.13.0-beta.0"
3+
4+
configuration = var.clusters["kind_zero"]
5+
}

‎tests/kubestack-starter-kind/config.auto.tfvars

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
clusters = {
2+
kind_zero = {
3+
# Settings for Apps-cluster
4+
apps = {
5+
name_prefix = "kind"
6+
base_domain = "infra.127.0.0.1.xip.io"
7+
8+
# clusters always have at least one control-plane node
9+
# uncommenting extra_nodes below will give you a cluster
10+
# with 3 control-plane nodes and 3 worker nodes
11+
# extra_nodes = "control-plane,control-plane,worker,worker,worker"
12+
}
13+
14+
# Settings for Ops-cluster
15+
ops = {
16+
# optionally reduce number of ops nodes
17+
# extra_nodes = "worker"
18+
}
19+
}
20+
}

‎tests/kubestack-starter-kind/manifests/bases/nginx/base/kustomization.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: ingress-nginx
4+
commonAnnotations:
5+
app.kubernetes.io/version: v0.40.2
6+
catalog.kubestack.com/heritage: kubestack.com/catalog/nginx
7+
catalog.kubestack.com/variant: base
8+
commonLabels:
9+
app.kubernetes.io/component: ingress-controller
10+
app.kubernetes.io/managed-by: kubestack
11+
app.kubernetes.io/name: nginx
12+
resources:
13+
- mandatory.yaml
14+
replicas:
15+
- name: ingress-nginx-controller
16+
count: 2

0 commit comments

Comments
 (0)