Skip to content

Commit d22a845

Browse files
balopatbalopat
authored
cloudbuild: fix docs generation + adds testing for GCB triggers (GoogleContainerTools#661)
refactor Dockerfile to multistage + makefile + --cache-from
1 parent dade08e commit d22a845

File tree

7 files changed

+245
-73
lines changed

7 files changed

+245
-73
lines changed

‎.gcloudignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
.github
2+
.idea
3+
out

‎Makefile

+74-25
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,15 @@ VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
2222
GOOS ?= $(shell go env GOOS)
2323
GOARCH = amd64
2424
BUILD_DIR ?= ./out
25+
DOCS_DIR ?= ./docs/generated
2526
ORG := github.com/GoogleContainerTools
2627
PROJECT := skaffold
2728
REPOPATH ?= $(ORG)/$(PROJECT)
2829
RELEASE_BUCKET ?= $(PROJECT)
30+
GSC_BUILD_PATH ?= gs://$(RELEASE_BUCKET)/builds/$(COMMIT)
31+
GSC_BUILD_LATEST ?= gs://$(RELEASE_BUCKET)/builds/latest
32+
GSC_RELEASE_PATH ?= gs://$(RELEASE_BUCKET)/releases/$(VERSION)
33+
GSC_RELEASE_LATEST ?= gs://$(RELEASE_BUCKET)/releases/latest
2934

3035
REMOTE_INTEGRATION ?= false
3136
GCP_PROJECT ?= k8s-skaffold
@@ -81,23 +86,62 @@ integration: install $(BUILD_DIR)/$(PROJECT)
8186
go test -v -tags integration $(REPOPATH)/integration -timeout 10m --remote=$(REMOTE_INTEGRATION)
8287

8388
.PHONY: release
84-
release: cross
85-
gsutil cp $(BUILD_DIR)/$(PROJECT)-* gs://$(RELEASE_BUCKET)/releases/$(VERSION)/
86-
gsutil cp $(BUILD_DIR)/$(PROJECT)-* gs://$(RELEASE_BUCKET)/latest/
89+
release: cross docs
90+
docker build \
91+
-f deploy/skaffold/Dockerfile \
92+
--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
93+
-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) .
94+
gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_RELEASE_PATH)/
95+
gsutil -m cp -r $(DOCS_DIR)/* $(GSC_RELEASE_PATH)/docs/
96+
gsutil -m cp -r $(GSC_RELEASE_PATH)/* $(GSC_RELEASE_LATEST)
97+
98+
.PHONY: release-in-docker
99+
release-in-docker:
100+
docker build \
101+
-f deploy/skaffold/Dockerfile \
102+
-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
103+
--target builder \
104+
.
105+
docker run \
106+
-v /var/run/docker.sock:/var/run/docker.sock \
107+
-v $(HOME)/.config/gcloud:/root/.config/gcloud \
108+
gcr.io/$(GCP_PROJECT)/skaffold-builder make -j release RELEASE_BUCKET=$(RELEASE_BUCKET) GCP_PROJECT=$(GCP_PROJECT)
109+
110+
.PHONY: release-build
111+
release-build: cross docs
112+
docker build \
113+
-f deploy/skaffold/Dockerfile \
114+
--cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \
115+
-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) .
116+
gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_BUILD_PATH)/
117+
gsutil -m cp -r $(DOCS_DIR)/* $(GSC_BUILD_PATH)/docs/
118+
gsutil -m cp -r $(GSC_BUILD_PATH)/* $(GSC_BUILD_LATEST)
119+
120+
.PHONY: release-build-in-docker
121+
release-build-in-docker:
122+
docker build \
123+
-f deploy/skaffold/Dockerfile \
124+
-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
125+
--target builder \
126+
.
127+
docker run \
128+
-v /var/run/docker.sock:/var/run/docker.sock \
129+
-v $(HOME)/.config/gcloud:/root/.config/gcloud \
130+
gcr.io/$(GCP_PROJECT)/skaffold-builder make -j release-build RELEASE_BUCKET=$(RELEASE_BUCKET) GCP_PROJECT=$(GCP_PROJECT)
87131

88132
.PHONY: clean
89133
clean:
90-
rm -rf $(BUILD_DIR)
134+
rm -rf $(BUILD_DIR) $(DOCS_DIR)
91135

92136
.PHONY: integration-in-docker
93137
integration-in-docker:
94138
docker build \
95139
-f deploy/skaffold/Dockerfile \
140+
--target integration \
96141
-t gcr.io/$(GCP_PROJECT)/skaffold-integration .
97142
docker run \
98143
-v /var/run/docker.sock:/var/run/docker.sock \
99144
-v $(HOME)/.config/gcloud:/root/.config/gcloud \
100-
-v $(PWD):/go/src/$(REPOPATH) \
101145
-v $(GOOGLE_APPLICATION_CREDENTIALS):$(GOOGLE_APPLICATION_CREDENTIALS) \
102146
-e REMOTE_INTEGRATION=true \
103147
-e DOCKER_CONFIG=/root/.docker \
@@ -106,23 +150,28 @@ integration-in-docker:
106150

107151
.PHONY: docs
108152
docs:
109-
rm -rf docs/generated
110-
mkdir -p docs/generated
111-
cp -R docs/css docs/generated/
112-
docker run -v $(PWD):/documents/ asciidoctor/docker-asciidoctor \
113-
asciidoctor \
114-
-a version="$(VERSION)" \
115-
-a commit="$(COMMIT)" \
116-
-a data-uri \
117-
-d book \
118-
-D docs/generated/ \
119-
docs/index.adoc
120-
docker run -v $(PWD):/documents/ asciidoctor/docker-asciidoctor \
121-
asciidoctor-pdf \
122-
-a version="$(VERSION)" \
123-
-a commit="$(COMMIT)" \
124-
-a allow-uri-read \
125-
-d book \
126-
-a pdf \
127-
-D docs/generated/ \
128-
docs/index.adoc
153+
hack/build_docs.sh $(VERSION) $(COMMIT)
154+
155+
.PHONY: docs-in-docker
156+
docs-in-docker:
157+
docker build \
158+
-f deploy/skaffold/Dockerfile \
159+
-t skaffold-builder \
160+
--target builder \
161+
.
162+
docker run \
163+
-v $(PWD):/go/src/$(REPOPATH) \
164+
skaffold-builder make docs
165+
166+
.PHONY: submit-build-trigger
167+
submit-build-trigger:
168+
gcloud container builds submit . \
169+
--config=deploy/cloudbuild.yaml \
170+
--substitutions="_RELEASE_BUCKET=$(RELEASE_BUCKET),COMMIT_SHA=$(COMMIT)"
171+
172+
.PHONY: submit-release-trigger
173+
submit-release-trigger:
174+
gcloud container builds submit . \
175+
--config=deploy/cloudbuild-release.yaml \
176+
--substitutions="_RELEASE_BUCKET=$(RELEASE_BUCKET),TAG_NAME=$(VERSION)"
177+

‎deploy/cloudbuild-release.yaml

+40-17
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,44 @@
11
# using default substitutions, provided by Google Container Builder
22
# see: https://cloud.google.com/container-builder/docs/configuring-builds/substitute-variable-values#using_default_substitutions
33
steps:
4-
# Build the container that will do the go build.
4+
5+
# Download the latest version of the builder image
56
- name: 'gcr.io/cloud-builders/docker'
6-
args: ['build', '-t', 'gcr.io/k8s-skaffold/skaffold:$TAG_NAME', '-f', 'deploy/skaffold/Dockerfile', '.']
7-
# Do the go build.
8-
- name: 'gcr.io/k8s-skaffold/skaffold:$TAG_NAME'
9-
args: ['make', 'cross', 'docs']
10-
# Upload to GCS.
11-
- name: 'gcr.io/cloud-builders/gsutil'
12-
args: ['cp', '-r', 'out/skaffold-*', 'gs://skaffold/releases/$TAG_NAME/']
13-
# Upload docs to GCS.
14-
- name: 'gcr.io/cloud-builders/gsutil'
15-
args: ['cp', '-r', 'docs/generated/*', 'gs://skaffold/releases/$TAG_NAME/docs/']
16-
# Bump the latest build
17-
# Upload to GCS.
18-
- name: 'gcr.io/cloud-builders/gsutil'
19-
args: ['cp', '-r', 'gs://skaffold/releases/$TAG_NAME/*', 'gs://skaffold/releases/latest/']
20-
images: ['gcr.io/k8s-skaffold/skaffold:$TAG_NAME']
21-
timeout: 20m
7+
entrypoint: 'bash'
8+
args:
9+
- '-c'
10+
- |
11+
docker pull gcr.io/$PROJECT_ID/skaffold-builder:latest || exit 0
12+
# until https://github.com/GoogleCloudPlatform/cloud-builders/issues/253 is fixed
13+
14+
# Rebuild the builder image if necessary
15+
- name: 'gcr.io/cloud-builders/docker'
16+
args:
17+
- 'build'
18+
- '-t'
19+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
20+
- '-f'
21+
- 'deploy/skaffold/Dockerfile'
22+
- '--cache-from'
23+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
24+
- '--target'
25+
- 'builder'
26+
- '.'
27+
28+
# Do the go build & push the results to GCS
29+
- name: 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
30+
args:
31+
- 'make'
32+
- '-j'
33+
- 'release'
34+
- 'RELEASE_BUCKET=$_RELEASE_BUCKET'
35+
- 'GCP_PROJECT=$PROJECT_ID'
36+
37+
images:
38+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
39+
- 'gcr.io/$PROJECT_ID/skaffold:$TAG_NAME'
40+
41+
options:
42+
machineType: 'N1_HIGHCPU_8'
43+
44+
timeout: 1200s

‎deploy/cloudbuild.yaml

+40-17
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,44 @@
11
# using default substitutions, provided by Google Container Builder
22
# see: https://cloud.google.com/container-builder/docs/configuring-builds/substitute-variable-values#using_default_substitutions
33
steps:
4-
# Build the container that will do the go build.
4+
5+
# Download the latest version of the builder image
56
- name: 'gcr.io/cloud-builders/docker'
6-
args: ['build', '-t', 'gcr.io/k8s-skaffold/skaffold:$COMMIT_SHA', '-f', 'deploy/skaffold/Dockerfile', '.']
7-
# Do the go build.
8-
- name: 'gcr.io/k8s-skaffold/skaffold:$COMMIT_SHA'
9-
args: ['make', 'cross', 'docs']
10-
# Upload to GCS.
11-
- name: 'gcr.io/cloud-builders/gsutil'
12-
args: ['cp', '-r', 'out/skaffold-*', 'gs://skaffold/builds/$COMMIT_SHA/']
13-
# Upload docs to GCS.
14-
- name: 'gcr.io/cloud-builders/gsutil'
15-
args: ['cp', '-r', 'docs/generated/*', 'gs://skaffold/builds/$COMMIT_SHA/docs/']
16-
# Bump the latest build
17-
# Upload to GCS.
18-
- name: 'gcr.io/cloud-builders/gsutil'
19-
args: ['cp', '-r', 'gs://skaffold/builds/$COMMIT_SHA/*', 'gs://skaffold/builds/latest/']
20-
images: ['gcr.io/k8s-skaffold/skaffold:$COMMIT_SHA']
21-
timeout: 20m
7+
entrypoint: 'bash'
8+
args:
9+
- '-c'
10+
- |
11+
docker pull gcr.io/$PROJECT_ID/skaffold-builder:latest || exit 0
12+
# until https://github.com/GoogleCloudPlatform/cloud-builders/issues/253 is fixed
13+
14+
# Rebuild the builder image if necessary
15+
- name: 'gcr.io/cloud-builders/docker'
16+
args:
17+
- 'build'
18+
- '-t'
19+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
20+
- '-f'
21+
- 'deploy/skaffold/Dockerfile'
22+
- '--cache-from'
23+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
24+
- '--target'
25+
- 'builder'
26+
- '.'
27+
28+
# Do the go build & push the results to GCS
29+
- name: 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
30+
args:
31+
- 'make'
32+
- '-j'
33+
- 'release-build'
34+
- 'RELEASE_BUCKET=$_RELEASE_BUCKET'
35+
- 'GCP_PROJECT=$PROJECT_ID'
36+
37+
images:
38+
- 'gcr.io/$PROJECT_ID/skaffold-builder:latest'
39+
- 'gcr.io/$PROJECT_ID/skaffold:$COMMIT_SHA'
40+
41+
options:
42+
machineType: 'N1_HIGHCPU_8'
43+
44+
timeout: 1200s

‎deploy/skaffold/Dockerfile

+42-14
Original file line numberDiff line numberDiff line change
@@ -12,23 +12,11 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# Second stage is used to install the binaries that we need
16-
FROM gcr.io/gcp-runtimes/ubuntu_16_0_4
15+
FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 as runtime_deps
1716

1817
RUN apt-get update && \
1918
apt-get install --no-install-recommends --no-install-suggests -y \
20-
ca-certificates \
21-
curl \
22-
build-essential \
23-
git \
24-
gcc \
25-
python-dev \
26-
python-setuptools \
27-
lsb-release
28-
29-
COPY --from=golang:1.10 /usr/local/go /usr/local/go
30-
ENV PATH /usr/local/go/bin:/go/bin:$PATH
31-
ENV GOPATH /go/
19+
python-dev
3220

3321
ENV KUBECTL_VERSION v1.10.0
3422
RUN curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
@@ -60,8 +48,48 @@ RUN curl -LO https://github.com/kubernetes-sigs/kustomize/releases/download/v${K
6048

6149
ENV PATH /usr/local/go/bin:/go/bin:/google-cloud-sdk/bin:$PATH
6250

51+
FROM runtime_deps as builder
52+
53+
ENV ASCIIDOCTOR_VERSION=1.5.7.1
54+
ENV ASCIIDOCTOR_PDF_VERSION=1.5.0.alpha.16
55+
RUN apt-get install --no-install-recommends --no-install-suggests -y \
56+
ca-certificates \
57+
curl \
58+
build-essential \
59+
git \
60+
gcc \
61+
python-setuptools \
62+
lsb-release \
63+
software-properties-common \
64+
ruby-dev \
65+
apt-transport-https && \
66+
gem install --no-document "asciidoctor-pdf:${ASCIIDOCTOR_PDF_VERSION}" \
67+
asciidoctor:${ASCIIDOCTOR_VERSION} \
68+
pygments.rb rouge && \
69+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
70+
apt-key fingerprint 0EBFCD88 && \
71+
add-apt-repository \
72+
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
73+
xenial \
74+
edge" && \
75+
apt-get -y update && \
76+
apt-get -y install docker-ce=17.12.0~ce-0~ubuntu
77+
78+
COPY --from=golang:1.10 /usr/local/go /usr/local/go
79+
ENV PATH /usr/local/go/bin:/go/bin:$PATH
80+
ENV GOPATH /go/
81+
6382
WORKDIR /go/src/github.com/GoogleContainerTools/skaffold
83+
6484
COPY . .
85+
86+
FROM builder as integration
87+
6588
RUN make out/skaffold-linux-amd64 && mv out/skaffold-linux-amd64 /usr/bin/skaffold
6689

6790
CMD ["make", "integration"]
91+
92+
FROM runtime_deps as distribution
93+
94+
COPY --from=integration /usr/bin/skaffold /usr/bin/skaffold
95+

‎docs/index.adoc

+5
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@ Dan Lorenc <[email protected]>; Matt Rickard <[email protected]>; David Gageot <
33
:toc: left
44
:icons: font
55
:imagesdir: img
6+
ifdef::pdf[]
7+
:source-highlighter: rouge
8+
endif::[]
9+
ifndef::pdf[]
610
:source-highlighter: pygments
11+
endif::[]
712
:pygments-css: style
813
:stylesheet: css/skaffold.css
914
:linkcss: css/skaffold.css

‎hack/build_docs.sh

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2018 The Skaffold Authors
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -x
18+
19+
VERSION=$1
20+
COMMIT=$2
21+
22+
rm -rf docs/generated
23+
mkdir -p docs/generated
24+
cp -R docs/css docs/generated/
25+
26+
asciidoctor \
27+
-a version="$VERSION" \
28+
-a commit="$COMMIT" \
29+
-a data-uri \
30+
-d book \
31+
-D docs/generated/ \
32+
docs/index.adoc
33+
34+
asciidoctor-pdf \
35+
-a version="$VERSION" \
36+
-a commit="$COMMIT" \
37+
-a allow-uri-read \
38+
-d book \
39+
-a pdf \
40+
-D docs/generated/ \
41+
docs/index.adoc

0 commit comments

Comments
 (0)