This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync module repo ➜ generate TOC ➜ convert notebooks to myst ➜ normalize headings ➜ push to site repo | |
| on: | |
| repository_dispatch: | |
| types: [sync-module] | |
| permissions: | |
| contents: write | |
| jobs: | |
| sync-generate-convert-normalize-push: | |
| runs-on: ubuntu-latest | |
| env: | |
| SITE_REPO: ds-modules/ecc-textbook | |
| GH_PAT: ${{ secrets.ECC_TEXTBOOK_PAT }} | |
| steps: | |
| # ----------------------------------------------------------------------- | |
| # 1. Pull THIS repo (so we can commit back into it) | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # ----------------------------------------------------------------------- | |
| # 2. Remove the old copy of this module & pull fresh | |
| - name: Purge and update module folder | |
| run: | | |
| MODULE="${{ github.event.client_payload.folder_name }}" | |
| rm -rf "$MODULE" | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.client_payload.source_repo }} | |
| path: tmp-source | |
| fetch-depth: 0 | |
| - name: Copy in new files (skip .git/.github) | |
| run: | | |
| rsync -a --delete --exclude .git --exclude .github tmp-source/ "${{ github.event.client_payload.folder_name }}/" | |
| - name: Remove tmp-source checkout | |
| run: rm -rf tmp-source | |
| # ----------------------------------------------------------------------- | |
| # 3. Auto-generate _toc.yml (recursive scan) | |
| - name: Generate _toc.yml (one part per module) | |
| run: | | |
| python - <<'PY' | |
| import pathlib, yaml, re | |
| ROOT = pathlib.Path('.') # repo root (modules live at top level) | |
| # 🔹 Folder-name → Nice caption mapping | |
| PRETTY = { | |
| "ecc-biology" : "Biology", | |
| "ecc-statistics" : "Statistics", | |
| "ecc-cs9" : "CS 9", | |
| "ecc-calculus" : "Calculus", | |
| "ecc-business" : "Business", | |
| "ecc-sociology" : "Sociology", | |
| "ecc-ethnic-studies" : "Ethnic Studies", | |
| "ecc-chemistry" : "Chemistry", | |
| "ecc-child-development" : "Child Development" | |
| } | |
| toc_parts = [] | |
| for module_path in sorted(p for p in ROOT.iterdir() if p.is_dir()): | |
| module_key = module_path.name # e.g. ecc-biology | |
| caption = PRETTY.get( | |
| module_key, | |
| re.sub(r'^ecc-', '', module_key) # drop leading ecc- | |
| .replace('-', ' ') # dashes → spaces | |
| .title() # Title Case | |
| ) | |
| chapters = [] | |
| # Recursively collect every .ipynb (skip checkpoints) | |
| for nb in sorted(module_path.rglob('*.ipynb')): | |
| if 'ipynb_checkpoints' in nb.parts: | |
| continue | |
| rel = nb.relative_to(ROOT).with_suffix('') # strip .ipynb | |
| chapters.append({'file': str(rel).replace('\\', '/')}) | |
| if chapters: | |
| toc_parts.append({'caption': caption, 'chapters': chapters}) | |
| toc = { | |
| 'format': 'jb-book', | |
| 'root' : 'intro', # ensure intro.md exists | |
| 'parts' : toc_parts | |
| } | |
| pathlib.Path('_toc.yml').write_text( | |
| yaml.dump(toc, sort_keys=False), | |
| encoding='utf-8' | |
| ) | |
| PY | |
| # ----------------------------------------------------------------------- | |
| # 4. Commit _toc.yml + refreshed module | |
| - name: Commit back to notebooks repo | |
| run: | | |
| git config user.name "jonathanferrari" | |
| git config user.email "[email protected]" | |
| if [ -n "$(git status --porcelain)" ]; then | |
| git add . | |
| git commit -m "Auto-sync ${{ github.event.client_payload.folder_name }} + regenerate _toc.yml" | |
| git push | |
| else | |
| echo "Nothing to commit." | |
| fi | |
| # ----------------------------------------------------------------------- | |
| # 5. Convert ALL notebooks → MyST Markdown | |
| - name: Install conversion tools | |
| run: pip install jupytext==1.* | |
| - name: Checkout site repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.SITE_REPO }} | |
| path: site | |
| token: ${{ env.GH_PAT }} | |
| fetch-depth: 0 | |
| - name: Convert & copy notebooks (with debug) | |
| run: | | |
| set -euo pipefail | |
| echo "==> Preparing site/modules" | |
| mkdir -p site/modules | |
| declare -a fail_list=() | |
| # Skip ipynb_checkpoints; iterate over every .ipynb file | |
| while IFS= read -r nb; do | |
| md="site/modules/${nb%.ipynb}.md" | |
| mkdir -p "$(dirname "$md")" | |
| echo "Converting $nb → $md" | |
| if ! jupytext "$nb" --to myst -o "$md"; then | |
| echo "::error ::Failed to convert $nb" | |
| fail_list+=("$nb") | |
| fi | |
| done < <( | |
| find . -type d -name 'ipynb_checkpoints' -prune -o \ | |
| -type f -name '*.ipynb' -print | |
| ) | |
| # Stop the job if anything failed, printing a neat summary | |
| if [ ${#fail_list[@]} -ne 0 ]; then | |
| echo "-----------------------------------------------------------------" | |
| echo "The following notebooks could NOT be converted:" | |
| printf ' - %s\n' "${fail_list[@]}" | |
| echo "-----------------------------------------------------------------" | |
| exit 1 | |
| fi | |
| # --------------------------------------------------------------- | |
| # Only reached if every conversion succeeded | |
| echo "Copying book-level files" | |
| cp _toc.yml site/modules/_toc.yml | |
| echo "Copied _toc.ym" | |
| # ----------------------------------------------------------------------- | |
| # 6. Normalize heading levels in the generated Markdown | |
| - name: Normalise Markdown heading levels | |
| run: | | |
| # -print0 / -0 keeps filenames with spaces whole | |
| find site -type f -name '*.md' -print0 \ | |
| | xargs -0 python .github/scripts/normalize-headings.py -v | |
| # ----------------------------------------------------------------------- | |
| # 7. Push to Repo | |
| - name: Commit to site repo | |
| run: | | |
| cd site | |
| git config user.name "jonathanferrari" | |
| git config user.email "[email protected]" | |
| if [ -n "$(git status --porcelain)" ]; then | |
| git add . | |
| git commit -m "Sync from notebooks repo" | |
| git push | |
| else | |
| echo "Site repo already up to date." | |
| fi |