general: migrate to src/ package structure, update CI configs

Author: karlicoss

.ci/release

@@ -21,15 +21,15 @@ import shutil
 
 is_ci = os.environ.get('CI') is not None
 
-def main():
+def main() -> None:
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument('--test', action='store_true', help='use test pypi')
     args = p.parse_args()
 
     extra = []
     if args.test:
-        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+        extra.extend(['--repository', 'testpypi'])
 
     root = Path(__file__).absolute().parent.parent
     os.chdir(root) # just in case

.ci/run

@@ -1,24 +1,40 @@
-#!/bin/bash -eu
+#!/bin/bash
+set -eu
 
 cd "$(dirname "$0")"
-cd ..
+cd .. # git root
 
 if ! command -v sudo; then
-    # CI or Docker sometimes don't have it, so useful to have a dummy
+    # CI or Docker sometimes doesn't have it, so useful to have a dummy
     function sudo {
         "$@"
     }
 fi
 
 if [ -n "${CI-}" ]; then
     # install OS specific stuff here
-    if [[ "$OSTYPE" == "darwin"* ]]; then
+    case "$OSTYPE" in
+    darwin*) 
         # macos
         :
-    else
+        ;;
+    cygwin* | msys* | win*)
+        # windows
         :
-    fi
+        ;;
+    *)
+        # must be linux?
+        :
+        ;;
+    esac
+fi
+
+
+PY_BIN="python3"
+# some systems might have python pointing to python3
+if ! command -v python3 &> /dev/null; then
+    PY_BIN="python"
 fi
 
-pip3 install --user tox
-tox
+"$PY_BIN" -m pip install --user tox
+"$PY_BIN" -m tox --parallel --parallel-live "$@"

.github/workflows/main.yml

@@ -10,27 +10,32 @@ on:
   pull_request: # needed to trigger on others' PRs
   # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
   workflow_dispatch: # needed to trigger workflows manually
-  # todo cron?
+    # todo cron?
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
-env:
-  # useful for scripts & sometimes tests to know
-  CI: true
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-rc.3']
+        # vvv just an example of excluding stuff from matrix
+        # exclude: [{platform: macos-latest, python-version: '3.6'}]
 
     runs-on: ${{ matrix.platform }}
 
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-
     - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }}
+      # hmm somehow only seems necessary for 3.11 on osx?? 
       run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH
 
     - uses: actions/setup-python@v4
@@ -41,12 +46,14 @@ jobs:
       with:
         submodules: recursive
 
-    # uncomment for SSH debugging
-    # - uses: mxschmitt/action-tmate@v3
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
-    - run: .ci/run
+    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
+    - run: bash .ci/run
 
-    - uses: actions/upload-artifact@v3
+    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
+      uses: actions/upload-artifact@v3
       with:
         name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy/
@@ -62,7 +69,7 @@ jobs:
 
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.7'
+        python-version: '3.8'
 
     - uses: actions/checkout@v3
       with:

Makefile

@@ -11,4 +11,4 @@ doc: cog
 ## Update files using cog.py
 cog: orgparse/__init__.py
 orgparse/__init__.py: README.rst
-	cd orgparse && cog.py -r __init__.py
+	cd src/orgparse && cog.py -r __init__.py

conftest.py

@@ -0,0 +1,38 @@
+# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly
+# without it, pytest can't discover the package root for some reason
+# also see https://github.com/karlicoss/pytest_namespace_pkgs for more
+
+import pathlib
+from typing import Optional
+
+import _pytest.main
+import _pytest.pathlib
+
+# we consider all dirs in repo/ to be namespace packages
+root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src'
+assert root_dir.exists(), root_dir
+
+# TODO assert it contains package name?? maybe get it via setuptools..
+
+namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()]
+
+# resolve_package_path is called from _pytest.pathlib.import_path
+# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem
+resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
+def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+    result = path  # search from the test file upwards
+    for parent in result.parents:
+        if str(parent) in namespace_pkg_dirs:
+            return parent
+    raise RuntimeError("Couldn't determine path for ", path)
+_pytest.pathlib.resolve_package_path = resolve_package_path
+
+
+# without patching, the orig function returns just a package name for some reason
+# (I think it's used as a sort of fallback)
+# so we need to point it at the absolute path properly
+# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
+search_pypath_orig = _pytest.main.search_pypath
+def search_pypath(module_name: str) -> str:
+    return str(root_dir)
+_pytest.main.search_pypath = search_pypath

pytest.ini

@@ -8,3 +8,6 @@ addopts =
 
   # otherwise it won't discover doctests
   --doctest-modules
+
+  # show all test durations (unless they are too short)
+  --durations=0

setup.py

@@ -4,10 +4,9 @@
 
 
 def main():
-    pkg = 'orgparse'
-    subpkgs = find_namespace_packages('.', include=(pkg + '.*',))
+    pkgs = find_namespace_packages('src')
+    pkg = min(pkgs)
 
-    import orgparse
     setup(
         name=pkg,
         use_scm_version={
@@ -18,22 +17,24 @@ def main():
 
         zip_safe=False,
 
-        packages=[pkg, *subpkgs],
+        packages=pkgs,
+        package_dir={'': 'src'},
         package_data={
             pkg: ['py.typed'], # todo need the rest as well??
             'orgparse.tests.data': ['*.org'],
         },
 
-        author=orgparse.__author__,
+        author='Takafumi Arakaki, Dmitrii Gerasimov',
         author_email='[email protected]',
         maintainer='Dima Gerasimov (@karlicoss)',
         maintainer_email='[email protected]',
 
         url='https://github.com/karlicoss/orgparse',
-        license=orgparse.__license__,
+        license='BSD License',
 
         description='orgparse - Emacs org-mode parser in Python',
-        long_description=orgparse.__doc__,
+        # TODO add it back later, perhaps via ast?
+        # long_description=orgparse.__doc__,
 
         keywords='org org-mode emacs',
         classifiers=[

orgparse/__init__.py renamed to src/orgparse/__init__.py

@@ -113,8 +113,6 @@
 
 from .node import parse_lines, OrgEnv, OrgNode  # todo basenode??
 
-__author__ = 'Takafumi Arakaki, Dmitrii Gerasimov'
-__license__ = 'BSD License'
 __all__ = ["load", "loads", "loadi"]
 
 

orgparse/date.py renamed to src/orgparse/date.py

@@ -13,7 +13,8 @@
 def load_data(path):
     """Load data from python file"""
     ns = {}  # type: ignore
-    exec(Path(path).read_text(), ns)
+    # read_bytes() and compile hackery to avoid encoding issues (e.g. see 05_tags)
+    exec(compile(Path(path).read_bytes(), path, 'exec'), ns)
     return ns['data']
 
 

orgparse/extra.py renamed to src/orgparse/extra.py

@@ -2,21 +2,39 @@
 minversion = 3.7
 # relies on the correct version of Python installed
 envlist = tests,mypy
+# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
+# hack to prevent .tox from crapping to the project directory
+toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox
 
 [testenv]
-passenv = CI,CI_*
+# TODO how to get package name from setuptools?
+package_name = "orgparse"
+passenv =
+# useful for tests to know they are running under ci
+  CI
+  CI_*
+# respect user's cache dirs to prevent tox from crapping into project dir
+  MYPY_CACHE_DIR
+  PYTHONPYCACHEPREFIX
 
+
+# note: --use-pep517 here is necessary for tox --parallel flag to work properly
+# otherwise it seems that it tries to modify .eggs dir in parallel and it fails
 [testenv:tests]
 commands =
-    pip install -e .[testing]
-    python -m pytest --ignore-glob='**/_py3compat.py' orgparse {posargs}
+    {envpython} -m pip install --use-pep517 -e .[testing]
+    # posargs allow test filtering, e.g. tox ... -- -k test_name
+    {envpython} -m pytest \
+        --pyargs {[testenv]package_name} \
+        {posargs}
+
 
 [testenv:mypy]
 commands =
-    pip install -e .[linting]
-    python -m mypy --install-types --non-interactive \
-                   orgparse                          \
-                   # txt report is a bit more convenient to view on CI
-                   --txt-report  .coverage.mypy      \
-                   --html-report .coverage.mypy      \
-                   {posargs}
+    {envpython} -m pip install --use-pep517 -e .[linting]
+    {envpython} -m mypy --install-types --non-interactive \
+        -p {[testenv]package_name}       \
+        # txt report is a bit more convenient to view on CI
+        --txt-report  .coverage.mypy     \
+        --html-report .coverage.mypy     \
+        {posargs}