Skip to content

Commit f20aa5a

Browse files
RajrahaneRajrahane
committed
Add Versioned Publish Images GH CI
Signed-off-by: Rajvaibhav Rahane <[email protected]>
1 parent 4454886 commit f20aa5a

File tree

5 files changed

+215
-3
lines changed

5 files changed

+215
-3
lines changed

‎.github/workflows/publish_remote_vector_version_images.yml

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
name: Publish Remote-Vector-Index-Builder Version Images to opensearchstaging
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
publish_latest:
7+
description: 'Publish latest tag'
8+
required: true
9+
type: boolean
10+
default: true
11+
publish_version:
12+
description: 'Publish version tag'
13+
required: true
14+
type: boolean
15+
default: true
16+
base_image_tag:
17+
description: 'Base image tag to use (default: faiss-base-snapshot)'
18+
required: false
19+
type: string
20+
default: 'faiss-base-snapshot'
21+
core_image_tag:
22+
description: 'Core image tag to use (default: core-snapshot)'
23+
required: false
24+
type: string
25+
default: 'core-snapshot'
26+
api_image_tag:
27+
description: 'API image tag to use (default: api-snapshot)'
28+
required: false
29+
type: string
30+
default: 'api-snapshot'
31+
remove_version_images:
32+
description: 'Remove version-tagged images after pushing'
33+
required: true
34+
type: boolean
35+
default: true
36+
remove_latest_images:
37+
description: 'Remove latest/snapshot-tagged images after pushing'
38+
required: true
39+
type: boolean
40+
default: true
41+
pull_request:
42+
branches:
43+
- "*"
44+
45+
permissions:
46+
id-token: write
47+
contents: write
48+
issues: write
49+
50+
jobs:
51+
build-and-publish-images:
52+
name: Build and Publish Remote-Vector-Index-Builder Docker Images
53+
if: github.repository == 'opensearch-project/remote-vector-index-builder'
54+
runs-on:
55+
group: selfhosted-gpu-runners
56+
labels: g62xlarge
57+
timeout-minutes: 60
58+
59+
steps:
60+
- name: Checkout code
61+
uses: actions/checkout@v4
62+
with:
63+
submodules: 'recursive'
64+
65+
- name: Get Version
66+
id: get-version
67+
run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
68+
69+
- name: Set Image Tags
70+
id: image-tags
71+
run: |
72+
# Set base image tag
73+
if [[ -n "${{ github.event.inputs.base_image_tag }}" ]]; then
74+
echo "base_tag=${{ github.event.inputs.base_image_tag }}" >> $GITHUB_OUTPUT
75+
elif [[ "${{ github.ref }}" =~ ^refs/heads/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
76+
echo "base_tag=faiss-base-latest" >> $GITHUB_OUTPUT
77+
else
78+
echo "base_tag=faiss-base-snapshot" >> $GITHUB_OUTPUT
79+
fi
80+
81+
# Set core image tag
82+
if [[ -n "${{ github.event.inputs.core_image_tag }}" ]]; then
83+
echo "core_tag=${{ github.event.inputs.core_image_tag }}" >> $GITHUB_OUTPUT
84+
elif [[ "${{ github.ref }}" =~ ^refs/heads/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
85+
echo "core_tag=core-latest" >> $GITHUB_OUTPUT
86+
else
87+
echo "core_tag=core-snapshot" >> $GITHUB_OUTPUT
88+
fi
89+
90+
# Set api image tag
91+
if [[ -n "${{ github.event.inputs.api_image_tag }}" ]]; then
92+
echo "api_tag=${{ github.event.inputs.api_image_tag }}" >> $GITHUB_OUTPUT
93+
elif [[ "${{ github.ref }}" =~ ^refs/heads/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
94+
echo "api_tag=api-latest" >> $GITHUB_OUTPUT
95+
else
96+
echo "api_tag=api-snapshot" >> $GITHUB_OUTPUT
97+
fi
98+
- name: Build Faiss Base Docker Image
99+
run : |
100+
docker build -f ./base_image/build_scripts/Dockerfile . \
101+
-t opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.base_tag }} \
102+
-t opensearchstaging/remote-vector-index-builder:faiss-base-${{ steps.get-version.outputs.VERSION }}
103+
- name: Build Core Docker Image
104+
run : |
105+
docker build -f ./remote_vector_index_builder/core/Dockerfile . \
106+
--build-arg BASE_IMAGE_TAG=${{ steps.image-tags.outputs.base_tag }} \
107+
-t opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.core_tag }} \
108+
-t opensearchstaging/remote-vector-index-builder:core-${{ steps.get-version.outputs.VERSION }}
109+
- name: Build API Docker Image
110+
run : |
111+
docker build -f ./remote_vector_index_builder/app/Dockerfile . \
112+
--build-arg CORE_IMAGE_TAG=${{ steps.image-tags.outputs.core_tag }} \
113+
-t opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.api_tag }} \
114+
-t opensearchstaging/remote-vector-index-builder:api-${{ steps.get-version.outputs.VERSION }}
115+
116+
- name: Get Approvers
117+
id: get_approvers
118+
run: |
119+
echo "approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
120+
121+
# https://github.com/trstringer/manual-approval
122+
- name: Get Manual Workflow Approval
123+
uses: trstringer/manual-approval@v1
124+
with:
125+
secret: ${{ github.TOKEN }}
126+
approvers: ${{ steps.get_approvers.outputs.approvers }}
127+
minimum-approvals: 1
128+
exclude-workflow-initiator-as-approver: false
129+
issue-title: 'Publish Remote-Vector-Index-Builder Version v${{ steps.get-version.outputs.version }} Images to opensearchstaging '
130+
issue-body: 'Please approve or deny publishing Remote-Vector-Index-Builder Version v${{ steps.get-version.outputs.version }} Images to opensearchstaging **COMMIT**: ${{ github.sha }}.'
131+
132+
- name: Configure AWS Credentials
133+
uses: aws-actions/configure-aws-credentials@v4
134+
with:
135+
role-to-assume: ${{ secrets.REMOTE_VECTOR_DOCKER_ROLE }}
136+
aws-region: us-east-1
137+
138+
- name: Retrieve Values
139+
id: retrieve-values
140+
run: |
141+
DOCKERHUB_PASSWORD=`aws secretsmanager get-secret-value --secret-id jenkins-staging-dockerhub-credential --query SecretString --output text`
142+
echo "::add-mask::$DOCKERHUB_PASSWORD"
143+
echo "dockerhub-password=$DOCKERHUB_PASSWORD" >> $GITHUB_OUTPUT
144+
145+
- name: Login to DockerHub
146+
uses: docker/login-action@v3
147+
with:
148+
username: ${{ secrets.REMOTE_VECTOR_DOCKER_USERNAME }}
149+
password: ${{ steps.retrieve-values.outputs.dockerhub-password }}
150+
151+
- name: Push Docker Image
152+
run : |
153+
if ${{ github.event.inputs.publish_version }}; then
154+
echo "Pushing version-tagged images"
155+
docker push opensearchstaging/remote-vector-index-builder:faiss-base-${{ steps.get-version.outputs.VERSION }}
156+
docker push opensearchstaging/remote-vector-index-builder:core-${{ steps.get-version.outputs.VERSION }}
157+
docker push opensearchstaging/remote-vector-index-builder:api-${{ steps.get-version.outputs.VERSION }}
158+
else
159+
echo "Skipping version-tagged images"
160+
fi
161+
if ${{ github.event.inputs.publish_latest }}; then
162+
docker push opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.base_tag }}
163+
docker push opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.core_tag }}
164+
docker push opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.api_tag }}
165+
else
166+
echo "Skipping latest-tagged images"
167+
fi
168+
169+
- name: Cleanup Docker Images
170+
if: always()
171+
shell: bash
172+
run: |
173+
# Function to safely remove docker image
174+
function remove_image() {
175+
local image="$1"
176+
if docker image inspect "$image" >/dev/null 2>&1; then
177+
echo "Force Removing image: $image"
178+
docker rmi -f "$image" || true
179+
else
180+
echo "Image not found: $image"
181+
fi
182+
}
183+
184+
# Remove version-tagged images if flag is set
185+
if [[ "${{ github.event.inputs.remove_version_images }}" == "true" ]]; then
186+
echo "Removing version-tagged images"
187+
remove_image "opensearchstaging/remote-vector-index-builder:faiss-base-${{ steps.get-version.outputs.VERSION }}"
188+
remove_image "opensearchstaging/remote-vector-index-builder:core-${{ steps.get-version.outputs.VERSION }}"
189+
remove_image "opensearchstaging/remote-vector-index-builder:api-${{ steps.get-version.outputs.VERSION }}"
190+
else
191+
echo "Skipping removal of version-tagged images"
192+
fi
193+
194+
# Remove latest/snapshot-tagged images if flag is set
195+
if [[ "${{ github.event.inputs.remove_latest_images }}" == "true" ]]; then
196+
echo "Removing latest/snapshot-tagged images"
197+
remove_image "opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.base_tag }}"
198+
remove_image "opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.core_tag }}"
199+
remove_image "opensearchstaging/remote-vector-index-builder:${{ steps.image-tags.outputs.api_tag }}"
200+
else
201+
echo "Skipping removal of latest/snapshot-tagged images"
202+
fi
203+
204+
- name: Runner Cleanups
205+
if: always()
206+
uses: ./.github/actions/docker-cleanup
207+
with:
208+
workspace: ${{ github.workspace }}

‎VERSION

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1.0.0

‎e2e/api/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM opensearchstaging/remote-vector-index-builder:faiss-base-latest
1+
ARG BASE_IMAGE_TAG=faiss-base-snapshot
2+
FROM opensearchstaging/remote-vector-index-builder:${BASE_IMAGE_TAG}
23
USER root
34

45
WORKDIR /app

‎remote_vector_index_builder/app/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
# this file be licensed under the Apache-2.0 license or a
66
# compatible open source license.
77

8-
FROM opensearchstaging/remote-vector-index-builder:core-latest
8+
ARG CORE_IMAGE_TAG=core-snapshot
9+
FROM opensearchstaging/remote-vector-index-builder:${CORE_IMAGE_TAG}
910

1011
USER root
1112

‎remote_vector_index_builder/core/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM opensearchstaging/remote-vector-index-builder:faiss-base-latest
1+
ARG BASE_IMAGE_TAG=faiss-base-snapshot
2+
FROM opensearchstaging/remote-vector-index-builder:${BASE_IMAGE_TAG}
23

34
USER root
45

0 commit comments

Comments
 (0)