diff --git a/.github/workflows/main.yml b/.github/workflows/ibis-backends.yml similarity index 65% rename from .github/workflows/main.yml rename to .github/workflows/ibis-backends.yml index 62bf3fecfa4d..9b96f488eddc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/ibis-backends.yml @@ -1,5 +1,5 @@ # vim: filetype=yaml -name: CI +name: Backends on: push: @@ -9,59 +9,13 @@ on: branches: - master -jobs: - test_no_backends: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - python-version: - - "3.7" - - "3.9" - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: install python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: install poetry - run: pip install poetry - - - name: cache dependencies linux - uses: actions/cache@v2 - if: ${{ matrix.os == 'ubuntu-latest' }} - with: - path: ~/.cache/pypoetry - key: no-backends-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} - - - name: cache dependencies windows - uses: actions/cache@v2 - if: ${{ matrix.os == 'windows-latest' }} - with: - path: ~\AppData\Local\pypoetry - key: no-backends-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} - - - name: install ibis - run: poetry install - - - name: run tests - shell: bash - run: ./ci/run_tests.sh ibis/tests - - - name: publish test report - uses: actions/upload-artifact@v2 - if: success() || failure() - with: - name: no-backends-${{ matrix.os }}-${{ matrix.python-version }} - path: junit.xml +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +jobs: test_simple_backends: + name: ${{ matrix.backend.title }} ${{ matrix.os }} python-${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -74,24 +28,33 @@ jobs: - "3.9" backend: - name: csv + title: CSV - name: dask + title: Dask - name: dask + title: Dask Extra deps: - "dask@2021.2.0 --extras array --extras dataframe" - name: hdf5 + title: HDF5 - name: pandas + title: Pandas - name: parquet + title: Parquet - name: sqlite + title: SQLite exclude: - os: windows-latest python-version: "3.9" backend: name: hdf5 + title: HDF5 include: - os: windows-latest python-version: "3.8" backend: name: hdf5 + title: HDF5 steps: - name: checkout uses: actions/checkout@v2 @@ -145,6 +108,7 @@ jobs: path: junit.xml test_postgres: + name: PostgreSQL ubuntu-latest python-${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -215,6 +179,7 @@ jobs: path: junit.xml test_pyspark: + name: PySpark ${{ matrix.pyspark.version }} ubuntu-latest python-${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -227,7 +192,8 @@ jobs: jdk: "8" env: ARROW_PRE_0_15_IPC_FORMAT: 1 - - jdk: "11" + - version: "latest" + jdk: "11" env: ARROW_PRE_0_15_IPC_FORMAT: 0 exclude: @@ -256,13 +222,13 @@ jobs: - name: cache dependencies uses: actions/cache@v2 - if: ${{ matrix.pyspark.version == null }} + if: ${{ matrix.pyspark.version == 'latest' }} with: path: ~/.cache/pypoetry key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}-pyspark-${{ matrix.pyspark.version }} - name: pin pyspark - if: ${{ matrix.pyspark.version != null }} + if: ${{ matrix.pyspark.version != 'latest' }} run: poetry add --lock --optional "pyspark@${{ matrix.pyspark.version }}" - name: install ibis @@ -285,6 +251,7 @@ jobs: path: junit.xml test_impala: + name: Impala ubuntu-latest python-${{ matrix.python-version }} runs-on: ubuntu-latest env: IBIS_TEST_NN_HOST: localhost @@ -386,6 +353,7 @@ jobs: path: junit.xml test_mysql_clickhouse: + name: ${{ matrix.backend.title }} ubuntu-latest python-${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -394,8 +362,10 @@ jobs: - "3.7" - "3.9" backend: - - mysql - - clickhouse + - name: mysql + title: MySQL + - name: clickhouse + title: ClickHouse services: mysql: image: mariadb:10.4.12 @@ -428,20 +398,20 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pypoetry - key: ${{ matrix.backend }}-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + key: ${{ matrix.backend.name }}-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} - name: install ibis - run: poetry install --extras ${{ matrix.backend }} + run: poetry install --extras ${{ matrix.backend.name }} - name: download backend data run: poetry run python ci/datamgr.py download - name: install backend data - run: poetry run python ci/datamgr.py ${{ matrix.backend }} + run: poetry run python ci/datamgr.py ${{ matrix.backend.name }} - name: run tests env: - PYTEST_BACKENDS: ${{ matrix.backend }} + PYTEST_BACKENDS: ${{ matrix.backend.name }} run: ./ci/run_tests.sh - name: publish test report @@ -450,143 +420,3 @@ jobs: with: name: ${{ matrix.backend }}-${{ matrix.python-version }} path: junit.xml - - benchmarks: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: - - "3.9" - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: install python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: install system dependencies - run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev - - - name: install poetry - run: pip install poetry - - - name: cache dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pypoetry - key: benchmarks-impala-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} - - - name: install ibis - run: poetry install --extras impala - - - name: benchmark - run: | - set -euo pipefail - - poetry run asv machine --yes - poetry run asv dev - - docs: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: - - "3.9" - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: install system dependencies - run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev pandoc - - - name: install python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: install poetry - run: pip install poetry - - - name: cache dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pypoetry - key: docs-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} - - - name: install ibis - run: poetry install --extras all - - - name: build web - run: poetry run python -m pysuerga docs/web --target-path=docbuild - - - name: build docs - run: poetry run sphinx-build -b html docs/source docbuild/docs -W -T - - - name: Add config to docs - run: | - set -euo pipefail - - touch docbuild/.nojekyll - echo "ibis-project.org" > docbuild/CNAME - - - name: Push docs - if: ${{ github.event_name == 'push' }} - uses: cpina/github-action-push-to-another-repository@v1.3 - env: - API_TOKEN_GITHUB: ${{ secrets.IBIS_PROJECT_DOCS_PAT }} - with: - source-directory: docbuild - target-branch: master - destination-github-username: ibis-project - destination-repository-name: ibis-project.org - user-email: '41898282+github-actions[bot]@users.noreply.github.com' - - - name: Clean up doc build - run: rm -r docbuild - - conda_package: - # TODO: fully automate the conda-forge PR submission on release - runs-on: ubuntu-latest - strategy: - matrix: - python-version: - - "3.7" - defaults: - run: - shell: bash -l {0} - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: update recipe file - run: | - set -euo pipefail - - # TODO: this is a hack that should be fixed by automatically generating - # a recipe from pyproject.toml once we move to poetry - set -x - IBIS_PATH=`pwd` - sed -i "s|url:.*|path: $IBIS_PATH|g" ci/recipe/meta.yaml - IBIS_VERSION=$(grep -Po '(?<=^version = ").+(?=")' pyproject.toml) - sed -i "s/{{ version }}/$IBIS_VERSION/g" ci/recipe/meta.yaml - cat ci/recipe/meta.yaml - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - miniforge-version: latest - miniforge-variant: Mambaforge - channel-priority: strict - activate-environment: ibis - python-version: ${{ matrix.python-version }} - condarc-file: ci/condarc - - - name: install boa - run: mamba install boa - - - name: build recipe - run: conda mambabuild -c conda-forge --python "${{ matrix.python-version }}" ci/recipe/meta.yaml diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml new file mode 100644 index 000000000000..025449a6c334 --- /dev/null +++ b/.github/workflows/ibis-main.yml @@ -0,0 +1,212 @@ +# vim: filetype=yaml +name: Ibis + +on: + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + test_no_backends: + name: Test ${{ matrix.os }} python-${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: install poetry + run: pip install poetry + + - name: cache dependencies linux + uses: actions/cache@v2 + if: ${{ matrix.os == 'ubuntu-latest' }} + with: + path: ~/.cache/pypoetry + key: no-backends-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: cache dependencies windows + uses: actions/cache@v2 + if: ${{ matrix.os == 'windows-latest' }} + with: + path: ~\AppData\Local\pypoetry + key: no-backends-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: install ibis + run: poetry install + + - name: run tests + shell: bash + run: ./ci/run_tests.sh ibis/tests + + - name: publish test report + uses: actions/upload-artifact@v2 + if: success() || failure() + with: + name: no-backends-${{ matrix.os }}-${{ matrix.python-version }} + path: junit.xml + + benchmarks: + name: Benchmark ${{ matrix.os }} python-${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.9" + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: install system dependencies + run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev + + - name: install poetry + run: pip install poetry + + - name: cache dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry + key: benchmarks-impala-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: install ibis + run: poetry install --extras impala + + - name: benchmark + run: | + set -euo pipefail + + poetry run asv machine --yes + poetry run asv dev + + docs: + name: Docs ${{ matrix.os }} python-${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.9" + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: install system dependencies + run: sudo apt-get install -qq -y build-essential cmake krb5-config python-dev libkrb5-dev libboost-all-dev pandoc + + - name: install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: install poetry + run: pip install poetry + + - name: cache dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry + key: docs-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: install ibis + run: poetry install --extras all + + - name: build web + run: poetry run python -m pysuerga docs/web --target-path=docbuild + + - name: build docs + run: poetry run sphinx-build -b html docs/source docbuild/docs -W -T + + - name: Add config to docs + run: | + set -euo pipefail + + touch docbuild/.nojekyll + echo "ibis-project.org" > docbuild/CNAME + + - name: Push docs + if: ${{ github.event_name == 'push' }} + uses: cpina/github-action-push-to-another-repository@v1.3 + env: + API_TOKEN_GITHUB: ${{ secrets.IBIS_PROJECT_DOCS_PAT }} + with: + source-directory: docbuild + target-branch: master + destination-github-username: ibis-project + destination-repository-name: ibis-project.org + user-email: '41898282+github-actions[bot]@users.noreply.github.com' + + - name: Clean up doc build + run: rm -r docbuild + + conda_package: + # TODO: fully automate the conda-forge PR submission on release + name: Conda Package ${{ matrix.os }} python-${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.7" + defaults: + run: + shell: bash -l {0} + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: update recipe file + run: | + set -euo pipefail + + # TODO: this is a hack that should be fixed by automatically generating + # a recipe from pyproject.toml once we move to poetry + set -x + IBIS_PATH=`pwd` + sed -i "s|url:.*|path: $IBIS_PATH|g" ci/recipe/meta.yaml + IBIS_VERSION=$(grep -Po '(?<=^version = ").+(?=")' pyproject.toml) + sed -i "s/{{ version }}/$IBIS_VERSION/g" ci/recipe/meta.yaml + cat ci/recipe/meta.yaml + + - uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + miniforge-version: latest + miniforge-variant: Mambaforge + channel-priority: strict + activate-environment: ibis + python-version: ${{ matrix.python-version }} + condarc-file: ci/condarc + + - name: install boa + run: mamba install boa + + - name: build recipe + run: conda mambabuild -c conda-forge --python "${{ matrix.python-version }}" ci/recipe/meta.yaml diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml index 535710467808..56447db6bf2e 100644 --- a/.github/workflows/test-report.yml +++ b/.github/workflows/test-report.yml @@ -1,7 +1,7 @@ name: Test Report on: workflow_run: - workflows: ['CI'] + workflows: ['Ibis', 'Backends'] types: - completed jobs: diff --git a/poetry.lock b/poetry.lock index 1974c2b00a9a..803aeda90ca7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1914,7 +1914,7 @@ visualization = ["graphviz"] [metadata] lock-version = "1.1" python-versions = ">=3.7.1,<4" -content-hash = "8c46ccb5a97d63e2436a034017837ffdbbdda3ac1e2ac8320c5eb785e5f54323" +content-hash = "3c3ad78683d6162a04fc2bef4bf57a55c573fb2a8e9a083c1e1531ec3bd5b204" [metadata.files] alabaster = [ @@ -2159,9 +2159,6 @@ coverage = [ {file = "coverage-6.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e14bceb1f3ae8a14374be2b2d7bc12a59226872285f91d66d301e5f41705d4d6"}, {file = "coverage-6.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0147f7833c41927d84f5af9219d9b32f875c0689e5e74ac8ca3cb61e73a698f9"}, {file = "coverage-6.1.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b1d0a1bce919de0dd8da5cff4e616b2d9e6ebf3bd1410ff645318c3dd615010a"}, - {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae6de0e41f44794e68d23644636544ed8003ce24845f213b24de097cbf44997f"}, - {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2797ed7a7e883b9ab76e8e778bb4c859fc2037d6fd0644d8675e64d58d1653"}, - {file = "coverage-6.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c40966b683d92869b72ea3c11fd6b99a091fd30e12652727eca117273fc97366"}, {file = "coverage-6.1.1-cp39-cp39-win32.whl", hash = "sha256:a11a2c019324fc111485e79d55907e7289e53d0031275a6c8daed30690bc50c0"}, {file = "coverage-6.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d8b453764b9b26b0dd2afb83086a7c3f9379134e340288d2a52f8a91592394b"}, {file = "coverage-6.1.1-pp36-none-any.whl", hash = "sha256:3b270c6b48d3ff5a35deb3648028ba2643ad8434b07836782b1139cf9c66313f"}, @@ -2659,6 +2656,8 @@ prompt-toolkit = [ {file = "prompt_toolkit-3.0.21.tar.gz", hash = "sha256:27f13ff4e4850fe8f860b77414c7880f67c6158076a7b099062cc8570f1562e5"}, ] psycopg2 = [ + {file = "psycopg2-2.9.1-cp310-cp310-win32.whl", hash = "sha256:25615574419dd9bda6fdfdcd58afb22e721f5b807cb3d5e62f488c8acf8cb754"}, + {file = "psycopg2-2.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:e5a8ed9dbfca8dc162c4ada5ab017e10d5a66c542b4c73569f103fa5f342f498"}, {file = "psycopg2-2.9.1-cp36-cp36m-win32.whl", hash = "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854"}, {file = "psycopg2-2.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56"}, {file = "psycopg2-2.9.1-cp37-cp37m-win32.whl", hash = "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188"}, diff --git a/pyproject.toml b/pyproject.toml index e5b897d203d8..97be8587c269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ pandas = "^1.2.5" parsy = "^1.3.0" pytz = "^2021.1" regex = "^2021.7.6" -cytoolz = "^0.11" +cytoolz = { version = "^0.11", python = "<3.10" } toolz = "^0.11" clickhouse-driver = { version = ">=0.1,<0.3", optional = true } clickhouse-sqlalchemy = { version = "^0.1.4", optional = true } diff --git a/setup.py b/setup.py index b8d7f5e8b780..ccde0fbf15cb 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ install_requires = [ "atpublic>=2.3,<3.0", "cached_property>=1,<2", - "cytoolz>=0.11,<0.12", "multipledispatch>=0.6,<0.7", "numpy>=1,<2", "pandas>=1.2.5,<2.0.0", @@ -62,6 +61,7 @@ ] extras_require = { + ':python_version < "3.10"': ["cytoolz>=0.11,<0.12"], ':python_version < "3.8"': ["importlib-metadata>=4,<5"], "all": [ "clickhouse-driver>=0.1,<0.3",