Skip to content

Commit 37ebf22

Browse files
gianfrancomaurogianfrancomauro
committed
Update and maintain refcounts.dat
Related to python#127443 Add a script to check and update `refcounts.dat` and integrate it into CI workflows. * **Add `Doc/tools/check_refcounts.py`**: - Create a script to check if `refcounts.dat` is up-to-date and sorted. - Implement logic to compare `refcounts.dat` with the stable ABI. - Implement logic to check if `refcounts.dat` is alphabetically sorted. - Handle cases where input files do not exist or cannot be read. - Allow passing file paths as command-line arguments. - Print total counts of entries and mismatches for better insights. - Optionally auto-sort and save the `refcounts.dat` file. * **Add `Doc/tools/Makefile`**: - Add a target to run the `check_refcounts.py` script. * **Update `.github/workflows/build.yml`**: - Add a CI job to run the `check_refcounts.py` script. - Ensure the job runs when a new function is added to the Stable ABI. * **Update `.github/workflows/lint.yml`**: - Add a CI linter job to check if `refcounts.dat` is correctly formatted. --- For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/python/cpython/issues/127443?shareId=XXXX-XXXX-XXXX-XXXX).
1 parent 49f15d8 commit 37ebf22

File tree

4 files changed

+422
-0
lines changed

4 files changed

+422
-0
lines changed

‎.github/workflows/build.yml

+18
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,22 @@ jobs:
143143
if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME
144144
run: make check-c-globals
145145

146+
check_refcounts:
147+
name: 'Check if refcounts.dat is up to date'
148+
runs-on: ubuntu-24.04
149+
timeout-minutes: 30
150+
needs: check_source
151+
if: needs.check_source.outputs.run_tests == 'true'
152+
steps:
153+
- uses: actions/checkout@v4
154+
- uses: actions/setup-python@v5
155+
with:
156+
python-version: '3.x'
157+
- name: Install Dependencies
158+
run: sudo ./.github/workflows/posix-deps-apt.sh
159+
- name: Check refcounts.dat
160+
run: make -C Doc/tools check_refcounts
161+
146162
build_windows:
147163
name: >-
148164
Windows
@@ -554,6 +570,7 @@ jobs:
554570
- check-docs
555571
- check_autoconf_regen
556572
- check_generated_files
573+
- check_refcounts
557574
- build_macos
558575
- build_ubuntu
559576
- build_ubuntu_ssltests
@@ -590,6 +607,7 @@ jobs:
590607
&& '
591608
check_autoconf_regen,
592609
check_generated_files,
610+
check_refcounts,
593611
build_macos,
594612
build_ubuntu,
595613
build_ubuntu_ssltests,

‎.github/workflows/lint.yml

+14
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,17 @@ jobs:
2424
with:
2525
python-version: "3.x"
2626
- uses: pre-commit/[email protected]
27+
28+
check_refcounts_format:
29+
name: 'Check if refcounts.dat is correctly formatted'
30+
runs-on: ubuntu-24.04
31+
timeout-minutes: 30
32+
steps:
33+
- uses: actions/checkout@v4
34+
- uses: actions/setup-python@v5
35+
with:
36+
python-version: '3.x'
37+
- name: Install Dependencies
38+
run: sudo ./.github/workflows/posix-deps-apt.sh
39+
- name: Check refcounts.dat format
40+
run: make -C Doc/tools check_refcounts

‎Doc/tools/Makefile

+335
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
# Makefile for Python documentation tools
2+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
4+
# You can set these variables from the command line.
5+
PYTHON = python3
6+
VENVDIR = ./venv
7+
SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
8+
BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb
9+
JOBS = auto
10+
PAPER =
11+
SOURCES =
12+
DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py)
13+
REQUIREMENTS = requirements.txt
14+
SPHINXERRORHANDLING = -W
15+
16+
# Internal variables.
17+
PAPEROPT_a4 = -D latex_elements.papersize=a4paper
18+
PAPEROPT_letter = -D latex_elements.papersize=letterpaper
19+
20+
ALLSPHINXOPTS = -b $(BUILDER) \
21+
-d build/doctrees \
22+
-j $(JOBS) \
23+
$(PAPEROPT_$(PAPER)) \
24+
$(SPHINXOPTS) $(SPHINXERRORHANDLING) \
25+
. build/$(BUILDER) $(SOURCES)
26+
27+
.PHONY: help
28+
help:
29+
@echo "Please use \`make <target>' where <target> is one of"
30+
@echo " clean to remove build files"
31+
@echo " venv to create a venv with necessary tools"
32+
@echo " html to make standalone HTML files"
33+
@echo " gettext to generate POT files"
34+
@echo " htmlview to open the index page built by the html target in your browser"
35+
@echo " htmllive to rebuild and reload HTML files in your browser"
36+
@echo " htmlhelp to make HTML files and a HTML help project"
37+
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
38+
@echo " text to make plain text files"
39+
@echo " texinfo to make Texinfo file"
40+
@echo " epub to make EPUB files"
41+
@echo " changes to make an overview over all changed/added/deprecated items"
42+
@echo " linkcheck to check all external links for integrity"
43+
@echo " coverage to check documentation coverage for library and C API"
44+
@echo " doctest to run doctests in the documentation"
45+
@echo " pydoc-topics to regenerate the pydoc topics file"
46+
@echo " dist to create a \"dist\" directory with archived docs for download"
47+
@echo " check to run a check for frequent markup errors"
48+
49+
.PHONY: build
50+
build:
51+
-mkdir -p build
52+
# Look first for a Misc/NEWS file (building from a source release tarball
53+
# or old repo) and use that, otherwise look for a Misc/NEWS.d directory
54+
# (building from a newer repo) and use blurb to generate the NEWS file.
55+
@if [ -f ../Misc/NEWS ] ; then \
56+
echo "Using existing Misc/NEWS file"; \
57+
cp ../Misc/NEWS build/NEWS; \
58+
elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \
59+
if [ -d ../Misc/NEWS.d ]; then \
60+
echo "Building NEWS from Misc/NEWS.d with blurb"; \
61+
$(BLURB) merge -f build/NEWS; \
62+
else \
63+
echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \
64+
exit 1; \
65+
fi \
66+
else \
67+
echo ""; \
68+
echo "Missing the required blurb or sphinx-build tools."; \
69+
echo "Please run 'make venv' to install local copies."; \
70+
echo ""; \
71+
exit 1; \
72+
fi
73+
$(SPHINXBUILD) $(ALLSPHINXOPTS)
74+
@echo
75+
76+
.PHONY: html
77+
html: BUILDER = html
78+
html: build
79+
@echo "Build finished. The HTML pages are in build/html."
80+
81+
.PHONY: htmlhelp
82+
htmlhelp: BUILDER = htmlhelp
83+
htmlhelp: build
84+
@echo "Build finished; now you can run HTML Help Workshop with the" \
85+
"build/htmlhelp/pydoc.hhp project file."
86+
87+
.PHONY: latex
88+
latex: BUILDER = latex
89+
latex: build
90+
@echo "Build finished; the LaTeX files are in build/latex."
91+
@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
92+
"run these through (pdf)latex."
93+
94+
.PHONY: text
95+
text: BUILDER = text
96+
text: build
97+
@echo "Build finished; the text files are in build/text."
98+
99+
.PHONY: texinfo
100+
texinfo: BUILDER = texinfo
101+
texinfo: build
102+
@echo "Build finished; the python.texi file is in build/texinfo."
103+
@echo "Run \`make info' in that directory to run it through makeinfo."
104+
105+
.PHONY: epub
106+
epub: BUILDER = epub
107+
epub: build
108+
@echo "Build finished; the epub files are in build/epub."
109+
110+
.PHONY: changes
111+
changes: BUILDER = changes
112+
changes: build
113+
@echo "The overview file is in build/changes."
114+
115+
.PHONY: linkcheck
116+
linkcheck: BUILDER = linkcheck
117+
linkcheck:
118+
@$(MAKE) build BUILDER=$(BUILDER) || { \
119+
echo "Link check complete; look for any errors in the above output" \
120+
"or in build/$(BUILDER)/output.txt"; \
121+
false; }
122+
123+
.PHONY: coverage
124+
coverage: BUILDER = coverage
125+
coverage: build
126+
@echo "Coverage finished; see c.txt and python.txt in build/coverage"
127+
128+
.PHONY: doctest
129+
doctest: BUILDER = doctest
130+
doctest:
131+
@$(MAKE) build BUILDER=$(BUILDER) || { \
132+
echo "Testing of doctests in the sources finished, look at the" \
133+
"results in build/doctest/output.txt"; \
134+
false; }
135+
136+
.PHONY: pydoc-topics
137+
pydoc-topics: BUILDER = pydoc-topics
138+
pydoc-topics: build
139+
@echo "Building finished; now run this:" \
140+
"cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py"
141+
142+
.PHONY: gettext
143+
gettext: BUILDER = gettext
144+
gettext: SPHINXOPTS += -d build/doctrees-gettext
145+
gettext: build
146+
147+
.PHONY: htmlview
148+
htmlview: html
149+
$(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))"
150+
151+
.PHONY: htmllive
152+
htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild
153+
htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0
154+
htmllive: _ensure-sphinx-autobuild html
155+
156+
.PHONY: clean
157+
clean: clean-venv
158+
-rm -rf build/*
159+
160+
.PHONY: clean-venv
161+
clean-venv:
162+
rm -rf $(VENVDIR)
163+
164+
.PHONY: venv
165+
venv:
166+
@if [ -d $(VENVDIR) ] ; then \
167+
echo "venv already exists."; \
168+
echo "To recreate it, remove it first with \`make clean-venv'."; \
169+
else \
170+
echo "Creating venv in $(VENVDIR)"; \
171+
if $(UV) --version >/dev/null 2>&1; then \
172+
$(UV) venv $(VENVDIR); \
173+
VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \
174+
else \
175+
$(PYTHON) -m venv $(VENVDIR); \
176+
$(VENVDIR)/bin/python3 -m pip install --upgrade pip; \
177+
$(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \
178+
fi; \
179+
echo "The venv has been created in the $(VENVDIR) directory"; \
180+
fi
181+
182+
.PHONY: dist-no-html
183+
dist-no-html: dist-text dist-pdf dist-epub dist-texinfo
184+
185+
.PHONY: dist
186+
dist:
187+
rm -rf dist
188+
mkdir -p dist
189+
$(MAKE) dist-html
190+
$(MAKE) dist-text
191+
$(MAKE) dist-pdf
192+
$(MAKE) dist-epub
193+
$(MAKE) dist-texinfo
194+
195+
.PHONY: dist-html
196+
dist-html:
197+
# archive the HTML
198+
@echo "Building HTML..."
199+
mkdir -p dist
200+
rm -rf build/html
201+
find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \;
202+
$(MAKE) html
203+
cp -pPR build/html dist/python-$(DISTVERSION)-docs-html
204+
tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html
205+
bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar
206+
(cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html)
207+
rm -r dist/python-$(DISTVERSION)-docs-html
208+
rm dist/python-$(DISTVERSION)-docs-html.tar
209+
@echo "Build finished and archived!"
210+
211+
.PHONY: dist-text
212+
dist-text:
213+
# archive the text build
214+
@echo "Building text..."
215+
mkdir -p dist
216+
rm -rf build/text
217+
find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \;
218+
$(MAKE) text
219+
cp -pPR build/text dist/python-$(DISTVERSION)-docs-text
220+
tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text
221+
bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar
222+
(cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text)
223+
rm -r dist/python-$(DISTVERSION)-docs-text
224+
rm dist/python-$(DISTVERSION)-docs-text.tar
225+
@echo "Build finished and archived!"
226+
227+
.PHONY: dist-pdf
228+
dist-pdf:
229+
# archive the A4 latex
230+
@echo "Building LaTeX (A4 paper)..."
231+
mkdir -p dist
232+
rm -rf build/latex
233+
find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \;
234+
$(MAKE) latex PAPER=a4
235+
# remove zip & bz2 dependency on all-pdf,
236+
# as otherwise the full latexmk process is run twice.
237+
# ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References)
238+
-sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile
239+
(cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2)
240+
cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip
241+
cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2
242+
@echo "Build finished and archived!"
243+
244+
.PHONY: dist-epub
245+
dist-epub:
246+
# copy the epub build
247+
@echo "Building EPUB..."
248+
mkdir -p dist
249+
rm -rf build/epub
250+
rm -f dist/python-$(DISTVERSION)-docs.epub
251+
$(MAKE) epub
252+
cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
253+
@echo "Build finished and archived!"
254+
255+
.PHONY: dist-texinfo
256+
dist-texinfo:
257+
# archive the texinfo build
258+
@echo "Building Texinfo..."
259+
mkdir -p dist
260+
rm -rf build/texinfo
261+
find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \;
262+
$(MAKE) texinfo
263+
$(MAKE) info --directory=build/texinfo
264+
cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo
265+
tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo
266+
bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar
267+
(cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo)
268+
rm -r dist/python-$(DISTVERSION)-docs-texinfo
269+
rm dist/python-$(DISTVERSION)-docs-texinfo.tar
270+
@echo "Build finished and archived!"
271+
272+
.PHONY: _ensure-package
273+
_ensure-package: venv
274+
if $(UV) --version >/dev/null 2>&1; then \
275+
VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \
276+
else \
277+
$(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \
278+
fi
279+
280+
.PHONY: _ensure-pre-commit
281+
_ensure-pre-commit:
282+
$(MAKE) _ensure-package PACKAGE=pre-commit
283+
284+
.PHONY: _ensure-sphinx-autobuild
285+
_ensure-sphinx-autobuild:
286+
$(MAKE) _ensure-package PACKAGE=sphinx-autobuild
287+
288+
.PHONY: check
289+
check: _ensure-pre-commit
290+
$(VENVDIR)/bin/python3 -m pre_commit run --all-files
291+
292+
.PHONY: serve
293+
serve:
294+
@echo "The serve target was removed, use htmllive instead (see gh-80510)"
295+
296+
# Targets for daily automated doc build
297+
# By default, Sphinx only rebuilds pages where the page content has changed.
298+
# This means it doesn't always pick up changes to preferred link targets, etc
299+
# To ensure such changes are picked up, we build the published docs with
300+
# `-E` (to ignore the cached environment) and `-a` (to ignore already existing
301+
# output files)
302+
303+
# for development releases: always build
304+
.PHONY: autobuild-dev
305+
autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
306+
autobuild-dev:
307+
$(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
308+
309+
# for HTML-only rebuilds
310+
.PHONY: autobuild-dev-html
311+
autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
312+
autobuild-dev-html:
313+
$(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
314+
315+
# for stable releases: only build if not in pre-release stage (alpha, beta)
316+
# release candidate downloads are okay, since the stable tree can be in that stage
317+
.PHONY: autobuild-stable
318+
autobuild-stable:
319+
@case $(DISTVERSION) in *[ab]*) \
320+
echo "Not building; $(DISTVERSION) is not a release version."; \
321+
exit 1;; \
322+
esac
323+
@$(MAKE) autobuild-dev
324+
325+
.PHONY: autobuild-stable-html
326+
autobuild-stable-html:
327+
@case $(DISTVERSION) in *[ab]*) \
328+
echo "Not building; $(DISTVERSION) is not a release version."; \
329+
exit 1;; \
330+
esac
331+
@$(MAKE) autobuild-dev-html
332+
333+
.PHONY: check_refcounts
334+
check_refcounts:
335+
$(PYTHON) tools/check_refcounts.py

0 commit comments

Comments
 (0)