Skip to content

Commit 5432402

Browse files
HyukjinKwonHyukjinKwon
committed
[SPARK-52584][BUILD] Make build script to support preview releases in finalize step
### What changes were proposed in this pull request? This PR proposes to make release script to support preview releases as well. ### Why are the changes needed? To make the release easier. ### Does this PR introduce _any_ user-facing change? No, dev-only. ### How was this patch tested? Manually tested against spark-website. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #51291 from HyukjinKwon/SPARK-52584. Authored-by: Hyukjin Kwon <[email protected]> Signed-off-by: Hyukjin Kwon <[email protected]>
1 parent b63221c commit 5432402

File tree

1 file changed

+147
-63
lines changed

1 file changed

+147
-63
lines changed

‎dev/create-release/release-build.sh

Lines changed: 147 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,12 @@ if [[ "$1" == "finalize" ]]; then
168168
import re
169169
170170
release_version = "${RELEASE_VERSION}"
171-
newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version}</a></li>'
171+
is_preview = bool(re.search(r'-preview\d*$', release_version))
172+
base_version = re.sub(r'-preview\d*$', '', release_version)
173+
174+
stable_newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version}</a></li>'
175+
preview_newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version} preview</a></li>'
176+
172177
inserted = False
173178
174179
def parse_version(v):
@@ -183,29 +188,63 @@ with open("documentation.md") as f:
183188
lines = f.readlines()
184189
185190
with open("documentation.md", "w") as f:
186-
for line in lines:
187-
match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
188-
if not inserted and match:
189-
existing_version = match.group(1)
190-
if vercmp(release_version, existing_version) >= 0:
191-
f.write(newline + "\n")
191+
if is_preview:
192+
in_preview_section = False
193+
for i, line in enumerate(lines):
194+
if '<p>Documentation for preview releases:</p>' in line:
195+
in_preview_section = True
196+
f.write(line)
197+
continue
198+
199+
if in_preview_section and re.search(r'docs/\d+\.\d+\.\d+-preview\d*/', line):
200+
existing_version = re.search(r'docs/(\d+\.\d+\.\d+-preview\d*)/', line).group(1)
201+
202+
if existing_version == release_version:
203+
inserted = True # Already exists, don't add
204+
elif not inserted:
205+
base_existing = re.sub(r'-preview\d*$', '', existing_version)
206+
preview_num_existing = int(re.search(r'preview(\d*)', existing_version).group(1) or "0")
207+
preview_num_new = int(re.search(r'preview(\d*)', release_version).group(1) or "0")
208+
209+
if (vercmp(base_version, base_existing) > 0) or \
210+
(vercmp(base_version, base_existing) == 0 and preview_num_new >= preview_num_existing):
211+
f.write(preview_newline + "\n")
212+
inserted = True
213+
214+
f.write(line)
215+
continue
216+
217+
if in_preview_section and "</ul>" in line and not inserted:
218+
f.write(preview_newline + "\n")
192219
inserted = True
193-
f.write(line)
194-
if not inserted:
195-
f.write(newline + "\n")
220+
f.write(line)
221+
else:
222+
for line in lines:
223+
match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
224+
if not inserted and match:
225+
existing_version = match.group(1)
226+
if vercmp(release_version, existing_version) >= 0:
227+
f.write(stable_newline + "\n")
228+
inserted = True
229+
f.write(line)
230+
if not inserted:
231+
f.write(stable_newline + "\n")
196232
EOF
197233

198234
echo "Edited documentation.md"
199235

200236
# 2. Add download link to js/downloads.js
201-
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
202-
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
203-
NEW_PACKAGES="packagesV14"
204-
if [[ "$rel_maj" -ge 4 ]]; then
205-
NEW_PACKAGES="packagesV15"
206-
fi
237+
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
238+
echo "Skipping js/downloads.js for preview release: $RELEASE_VERSION"
239+
else
240+
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
241+
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
242+
NEW_PACKAGES="packagesV14"
243+
if [[ "$rel_maj" -ge 4 ]]; then
244+
NEW_PACKAGES="packagesV15"
245+
fi
207246

208-
python3 <<EOF
247+
python3 <<EOF
209248
import re
210249
211250
release_version = "${RELEASE_VERSION}"
@@ -254,13 +293,44 @@ with open("js/downloads.js", "w") as f:
254293
f.write(newline + "\n")
255294
EOF
256295

257-
echo "Edited js/downloads.js"
296+
echo "Edited js/downloads.js"
297+
fi
258298

259299
# 3. Add news post
260300
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
261301
FILENAME="news/_posts/${RELEASE_DATE}-spark-${RELEASE_VERSION//./-}-released.md"
262302
mkdir -p news/_posts
263-
cat > "$FILENAME" <<EOF
303+
304+
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
305+
BASE_VERSION="${RELEASE_VERSION%%-preview*}"
306+
cat > "$FILENAME" <<EOF
307+
---
308+
layout: post
309+
title: Preview release of Spark ${BASE_VERSION}
310+
categories:
311+
- News
312+
tags: []
313+
status: publish
314+
type: post
315+
published: true
316+
meta:
317+
_edit_last: '4'
318+
_wpas_done_all: '1'
319+
---
320+
To enable wide-scale community testing of the upcoming Spark ${BASE_VERSION} release, the Apache Spark community has posted a
321+
<a href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/">Spark ${RELEASE_VERSION} release</a>.
322+
This preview is not a stable release in terms of either API or functionality, but it is meant to give the community early
323+
access to try the code that will become Spark ${BASE_VERSION}. If you would like to test the release,
324+
please <a href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/">download</a> it, and send feedback using either
325+
<a href="https://spark.apache.org/community.html">mailing lists</a> or
326+
<a href="https://issues.apache.org/jira/browse/SPARK/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel">JIRA</a>.
327+
The documentation is available at the <a href="https://spark.apache.org/docs/${RELEASE_VERSION}/">link</a>.
328+
329+
We'd like to thank our contributors and users for their contributions and early feedback to this release. This release would not have been possible without you.
330+
EOF
331+
332+
else
333+
cat > "$FILENAME" <<EOF
264334
---
265335
layout: post
266336
title: Spark ${RELEASE_VERSION} released
@@ -276,16 +346,20 @@ meta:
276346
---
277347
We are happy to announce the availability of <a href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html" title="Spark Release ${RELEASE_VERSION}">Apache Spark ${RELEASE_VERSION}</a>! Visit the <a href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html" title="Spark Release ${RELEASE_VERSION}">release notes</a> to read about the new features, or <a href="{{site.baseurl}}/downloads.html">download</a> the release today.
278348
EOF
349+
fi
279350

280351
echo "Created $FILENAME"
281352

282353
# 4. Add release notes with Python to extract JIRA version ID
283-
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
284-
JIRA_PROJECT_ID=12315420
285-
JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
286-
JSON=$(curl -s "$JIRA_URL")
354+
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
355+
echo "Skipping JIRA release notes for preview release: $RELEASE_VERSION"
356+
else
357+
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
358+
JIRA_PROJECT_ID=12315420
359+
JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
360+
JSON=$(curl -s "$JIRA_URL")
287361

288-
VERSION_ID=$(python3 - <<EOF
362+
VERSION_ID=$(python3 - <<EOF
289363
import sys, json
290364
291365
release_version = "${RELEASE_VERSION}"
@@ -305,32 +379,32 @@ for v in versions:
305379
306380
print(version_id)
307381
EOF
308-
)
382+
)
309383

310-
if [[ -z "$VERSION_ID" ]]; then
311-
echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
312-
fi
384+
if [[ -z "$VERSION_ID" ]]; then
385+
echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
386+
fi
313387

314-
JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}"
388+
JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}"
315389

316-
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
317-
if [[ "$rel_patch" -eq 0 ]]; then
318-
ACKNOWLEDGE="patches and features to this release."
319-
BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It introduces new functionality and improvements. We encourage users to try it and provide feedback."
320-
else
321-
ACKNOWLEDGE="patches to this release."
322-
BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing security and correctness fixes. This release is based on the branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend all ${rel_maj}.${rel_min} users to upgrade to this stable release."
323-
fi
390+
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
391+
if [[ "$rel_patch" -eq 0 ]]; then
392+
ACKNOWLEDGE="patches and features to this release."
393+
BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It introduces new functionality and improvements. We encourage users to try it and provide feedback."
394+
else
395+
ACKNOWLEDGE="patches to this release."
396+
BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing security and correctness fixes. This release is based on the branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend all ${rel_maj}.${rel_min} users to upgrade to this stable release."
397+
fi
324398

325-
BODY+="
399+
BODY+="
326400
327401
You can find the list of resolved issues and detailed changes in the [JIRA release notes](${JIRA_LINK}).
328402
329403
We would like to acknowledge all community members for contributing ${ACKNOWLEDGE}"
330404

331-
FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
332-
mkdir -p releases/_posts
333-
cat > "$FILENAME" <<EOF
405+
FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
406+
mkdir -p releases/_posts
407+
cat > "$FILENAME" <<EOF
334408
---
335409
layout: post
336410
title: Spark Release ${RELEASE_VERSION}
@@ -347,40 +421,50 @@ meta:
347421
${BODY}
348422
EOF
349423

350-
echo "Created $FILENAME"
424+
echo "Created $FILENAME"
425+
fi
351426

352427
# 5. Build the website
353428
bundle install
354429
bundle exec jekyll build
355430

356-
# 6. Update latest symlink if minor/major release
357-
LINK_PATH="site/docs/latest"
358-
TARGET_DIR="site/docs/$RELEASE_VERSION"
431+
# 6. Update latest or preview symlink
359432
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
360-
if [[ "$rel_patch" -eq 0 ]]; then
361-
if [[ -L "$LINK_PATH" ]]; then
362-
CURRENT_TARGET=$(readlink "$LINK_PATH")
363-
else
364-
CURRENT_TARGET=""
365-
fi
366433

367-
if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
368-
IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
434+
if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
435+
LINK_PATH="site/docs/preview"
369436

370-
if [[ "$rel_maj" -gt "$cur_maj" ]]; then
371-
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
372-
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version increased)"
373-
elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
374-
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
375-
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version increased)"
437+
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
438+
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (preview release)"
439+
440+
else
441+
LINK_PATH="site/docs/latest"
442+
443+
if [[ "$rel_patch" -eq 0 ]]; then
444+
if [[ -L "$LINK_PATH" ]]; then
445+
CURRENT_TARGET=$(readlink "$LINK_PATH")
376446
else
377-
echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer major.minor, no change"
447+
CURRENT_TARGET=""
448+
fi
449+
450+
if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
451+
IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
452+
453+
if [[ "$rel_maj" -gt "$cur_maj" ]]; then
454+
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
455+
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version increased)"
456+
elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
457+
ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
458+
echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version increased)"
459+
else
460+
echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer major.minor, no change"
461+
fi
462+
else
463+
echo "No valid existing version target."
378464
fi
379465
else
380-
echo "No valid existing version target."
466+
echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
381467
fi
382-
else
383-
echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
384468
fi
385469

386470
git add .

0 commit comments

Comments
 (0)