Merge pull request #36 from ecmwf/feat/downstream_ci

first draft updating github workflows

Author: Oisin-M

.flake8

@@ -0,0 +1,13 @@
+[flake8]
+# Example to set max line length
+max-line-length = 88
+
+# Per-file ignore settings
+per-file-ignores =
+    # Ignore F401 (unused import) in __init__.py files
+    __init__.py: F401
+
+    # Ignore specific errors in the tests folder
+    # - F405: Undefined variable
+    # - F403: Wildcard import usage
+    tests/*: F405,F403

.github/workflows/cd.yml

@@ -0,0 +1,12 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+
+  pypi:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

.github/workflows/ci.yml

@@ -8,22 +8,15 @@ on:
 
 jobs:
 
-  quality_assurance:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - run: pip install ruff
-      - run: ruff check # check linting violations
-      - run: ruff format --check # check formatting
+  qa-pre-commit:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    secrets: inherit
 
-  setup_and_test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
-      - name: install earthkit-hydro
-        shell: bash -l {0}
-        run: pip install .[test]
-      - name: run tests
-        shell: bash -l {0}
-        run: pytest
+  test:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    secrets: inherit
+
+  # TODO: add back
+  # qa-python:
+  #     uses: ecmwf-actions/reusable-workflows/.github/workflows/ci-python.yml@v2
+  #     secrets: inherit

.github/workflows/downstream-ci.yml

@@ -0,0 +1,77 @@
+name: downstream-ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feat/add-earthkit-hydro
+    with:
+      earthkit-hydro: ecmwf/earthkit-hydro@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+      python_qa: true
+    secrets: inherit
+
+  # Run CI of private downstream packages on self-hosted runners
+  private-downstream-ci:
+    name: private-downstream-ci
+    needs: [downstream-ci]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci
+          payload: '{"earthkit-hydro": "ecmwf/earthkit-hydro@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feat/add-earthkit-hydro
+    with:
+      earthkit-hydro: ecmwf/earthkit-hydro@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
+
+  # Run CI of private downstream packages on HPC
+  private-downstream-ci-hpc:
+    name: private-downstream-ci-hpc
+    needs: [downstream-ci-hpc]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci-hpc
+          payload: '{"earthkit-hydro": "ecmwf/earthkit-hydro@${{ github.event.pull_request.head.sha || github.sha }}"}'

.github/workflows/label-public-pr.yml

@@ -0,0 +1,9 @@
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

.gitignore

@@ -1,3 +1,3 @@
 __pycache__/
 .ipynb_checkpoints/
-*.egg-info
+*.egg-info

.pre-commit-config.yaml

@@ -12,6 +12,18 @@ repos:
             args: [ --fix ]
           - id: ruff-format # fix formatting
             types_or: [ python, pyi, jupyter ]
+    - repo: https://github.com/psf/black
+      rev: 25.1.0
+      hooks:
+        - id: black
+    - repo: https://github.com/pycqa/isort
+      rev: 5.13.2
+      hooks:
+        - id: isort
+    - repo: https://github.com/pycqa/flake8
+      rev: 7.0.0
+      hooks:
+        - id: flake8
     - repo: https://github.com/pre-commit/pre-commit-hooks
       rev: v4.4.0
       hooks:

docs/notebooks/example.ipynb

@@ -370,7 +370,8 @@
    ],
    "source": [
     "precipitation_field = ekd.from_source(\n",
-    "    \"file\", \"/ec/ws4/tc/emos/work/cems/floods/efas/emos/efas/europe5km/202502/EUD2025021100/forcings/R06a.nc\"\n",
+    "    \"file\",\n",
+    "    \"/ec/ws4/tc/emos/work/cems/floods/efas/emos/efas/europe5km/202502/EUD2025021100/forcings/R06a.nc\",\n",
     ")[0].to_numpy()\n",
     "\n",
     "qplot.contourf(precipitation_field[::-1, :], style=style)"

pyproject.toml

@@ -6,12 +6,12 @@ build-backend = "setuptools.build_meta"
 name = "earthkit-hydro"
 requires-python = ">=3.8"
 authors = [
-    {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "[email protected]"},
-]
+    { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "[email protected]" }
+    ]
 maintainers = [
     {name = "Oisín M. Morrison", email = "[email protected]"},
-    {name = "Corentin Carton de Wiart", email = "[email protected]"},
-]
+    {name = "Corentin Carton de Wiart", email = "[email protected]"}
+    ]
 description = "A Python library for common hydrological functions"
 license = {file = "LICENSE"}
 classifiers = [
@@ -27,14 +27,11 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
-    "Topic :: Scientific/Engineering",
-]
+    "Topic :: Scientific/Engineering"
+    ]
 dynamic = ["version", "readme"]
 
-dependencies = [
-    "numpy",
-    "joblib"
-]
+dependencies = ["numpy", "joblib"]
 
 [project.urls]
     repository = "https://github.com/ecmwf/earthkit-hydro"
@@ -43,43 +40,52 @@ dependencies = [
 
 [project.optional-dependencies]
     readers = [
-        "earthkit-data",
-    ]
-    test = [
+        "earthkit-data"
+        ]
+    tests = [
         "pytest",
-        "pytest-cases",
-    ]
+        "pytest-cases"
+        ]
     dev = [
         "pytest",
         "pytest-cases",
-        "ruff",
         "pre-commit"
-    ]
+        ]
+    all = [
+        "earthkit-data",
+        "pytest",
+        "pytest-cases",
+        "pre-commit"
+        ]
+
+[tool.black]
+line-length = 88
+skip-string-normalization = false
+
+[tool.isort]
+profile = "black"  # Ensures compatibility with Black's formatting.
+line_length = 88  # Same as Black's line length for consistency.
 
 # Linting settings
 [tool.ruff]
-line-length = 120
-indent-width = 4
+line-length = 88
 
 [tool.ruff.format]
 quote-style = "double"
-indent-style = "space"
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = [
     "F401", # unused imports
-]
+    ]
 "tests/*" = [
     "F405", # variable may be undefined, or defined from star imports
-    "F403" # use of wildcard imports
-]
+    "F403", # use of wildcard imports
+    ]
 
 # Testing
 [tool.pytest]
 addopts = "--pdbcls=IPython.terminal.debugger:Pdb"
-testpaths = [
-    "tests",
-]
+testpaths = ["tests"]
 
 # Packaging/setuptools options
 [tool.setuptools]
@@ -94,8 +100,7 @@ exclude = ["tests"]
 
 [tool.setuptools_scm]
 write_to = "src/earthkit/hydro/_version.py"
-write_to_template = '''
-# Do not change! Do not track in version control!
+write_to_template = '''# Do not change! Do not track in version control!
 __version__ = "{version}"
 '''
 parentdir_prefix_version='earthkit-hydro-'  # get version from GitHub-like tarballs

src/earthkit/hydro/__init__.py

@@ -1,6 +1,12 @@
-from .readers import from_cama_downxy, from_cama_nextxy, from_d8, load_river_network, create_river_network
-from .river_network import RiverNetwork
 from .accumulation import flow_downstream
-from .movement import move_downstream, move_upstream
 from .catchment import find_catchments, find_subcatchments
 from .core import flow
+from .movement import move_downstream, move_upstream
+from .readers import (
+    create_river_network,
+    from_cama_downxy,
+    from_cama_nextxy,
+    from_d8,
+    load_river_network,
+)
+from .river_network import RiverNetwork