Remove upper bounds for dependencies in requirements.txt (#5733)

* Remove upper bounds for dependencies in requirements.txt

The stated reason for this pinning was to prevent CI breakage, and there
was a bot to help upgrade dependencies automatically. However, now that
the library is being retired and Depdendabot was, this upgrade process
is no longer happening.

Removing these upper bounds will make using the library in the future
much easier. To prevent CI breakage after upper bound removal, they can
be kept in a constraints file for use in CI. It would be up to end users
in the future to limit dependency conflicts if they encounter any.

I have checked the history, and as far as I can tell, none of these
upper pins exist to prevent a known breakage. Therefore, I moved them
all to the constraints file.

* Use constraints file in more places

I needed to add a new `requirements.in` file and an extra constraint so
`pip-compile` resolved all dependencies properly.

Author: aphedges

.github/workflows/ci.yml

@@ -21,8 +21,8 @@ env:
   TORCH_VERSION: 1.12.0
   # TORCH_CPU_INSTALL: conda install pytorch torchvision torchaudio cpuonly -c pytorch
   # TORCH_GPU_INSTALL: conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
-  TORCH_CPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
-  TORCH_GPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
+  TORCH_CPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu -c constraints.txt
+  TORCH_GPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113 -c constraints.txt
   # Change this to invalidate existing cache.
   CACHE_PREFIX: v11
   # Disable tokenizers parallelism because this doesn't help, and can cause issues in distributed tests.
@@ -150,7 +150,7 @@ jobs:
       id: virtualenv-cache
       with:
         path: .venv
-        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt') }}
+        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
@@ -276,7 +276,7 @@ jobs:
       id: virtualenv-cache
       with:
         path: .venv
-        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt') }}
+        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
@@ -377,7 +377,7 @@ jobs:
 
     - name: Install core package
       run: |
-        pip install $(ls dist/*.whl)${{ matrix.flavor }}
+        pip install $(ls dist/*.whl)${{ matrix.flavor }} -c constraints.txt
 
     - name: Download NLTK prerequisites
       run: |
@@ -484,7 +484,7 @@ jobs:
       id: virtualenv-cache
       with:
         path: .venv
-        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt') }}
+        key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
 
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'

Dockerfile

@@ -14,9 +14,10 @@ COPY allennlp/version.py allennlp/version.py
 COPY setup.py .
 COPY requirements.txt .
 COPY dev-requirements.txt .
+COPY constraints.txt .
 RUN touch allennlp/__init__.py \
     && touch README.md \
-    && pip install --no-cache-dir -e .[all]
+    && pip install --no-cache-dir -c constraints.txt -e .[all]
 
 # Now add the full package source and re-install just the package.
 COPY allennlp allennlp

Dockerfile.test

@@ -16,9 +16,10 @@ WORKDIR /stage/allennlp
 COPY allennlp/version.py allennlp/version.py
 COPY setup.py .
 COPY dev-requirements.txt .
+COPY constraints.txt .
 RUN touch allennlp/__init__.py \
     && touch README.md \
-    && pip install --no-cache-dir -e . -r dev-requirements.txt
+    && pip install --no-cache-dir -c constraints.txt -e . -r dev-requirements.txt
 
 # Now add the full package source and re-install just the package.
 COPY . .

Makefile

@@ -10,7 +10,7 @@ MD_DOCS_CONF_SRC = mkdocs-skeleton.yml
 MD_DOCS_TGT = site/
 MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING.md)
 
-TORCH_INSTALL = pip install torch torchvision
+TORCH_INSTALL = pip install torch torchvision -c constraints.txt
 DOCKER_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
 DOCKER_TEST_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
 
@@ -115,7 +115,7 @@ install :
 	$(TORCH_INSTALL)
 	pip install --upgrade pip
 	pip install pip-tools
-	pip-compile requirements.txt dev-requirements.txt -o final_requirements.txt --allow-unsafe --rebuild --verbose
+	pip-compile requirements.in -o final_requirements.txt --allow-unsafe --rebuild --verbose
 	pip install -e . -r final_requirements.txt
 	# These nltk packages are used by the 'checklist' module.
 	$(NLTK_DOWNLOAD_CMD)

constraints.txt

@@ -0,0 +1,24 @@
+################################
+###### Core dependencies #######
+################################
+torch<1.13.0
+torchvision<0.14.0
+cached-path<1.2.0
+spacy<3.4
+transformers<4.21
+filelock<3.8
+wandb<0.13.0
+
+# Protobuf is a dependency of wandb and tensorboard, but they are missing this pin.
+protobuf<4.0.0
+
+# Required so pip-compile can properly resolve the pydantic version
+inflect<6.0
+
+##################################################
+###### Extra dependencies for integrations #######
+##################################################
+# NOTE: we use a special trailing comment on each line to denote which extras
+# each package is needed by. For example, checklist is needed by the 'checklist' extra
+# that you install with 'pip install allennlp[checklist]'.
+checklist==0.0.11  # needed by: checklist

requirements.in

@@ -0,0 +1,3 @@
+-c constraints.txt
+-r dev-requirements.txt
+-r requirements.txt

requirements.txt

@@ -1,13 +1,13 @@
 ################################
 ###### Core dependencies #######
 ################################
-torch>=1.10.0,<1.13.0
-torchvision>=0.8.1,<0.14.0
-cached-path>=1.1.3,<1.2.0
+torch>=1.10.0
+torchvision>=0.8.1
+cached-path>=1.1.3
 fairscale==0.4.6
 jsonnet>=0.10.0 ; sys.platform != 'win32'
 nltk>=3.6.5
-spacy>=2.1.0,<3.4
+spacy>=2.1.0
 numpy>=1.21.4
 tensorboardX>=1.2
 requests>=2.28
@@ -16,14 +16,14 @@ h5py>=3.6.0
 scikit-learn>=1.0.1
 scipy>=1.7.3
 pytest>=6.2.5
-transformers>=4.1,<4.21
+transformers>=4.1
 sentencepiece>=0.1.96
 dataclasses;python_version<'3.7'
-filelock>=3.3,<3.8
+filelock>=3.3
 lmdb>=1.2.1
 more-itertools>=8.12.0
 termcolor==1.1.0
-wandb>=0.10.0,<0.13.0
+wandb>=0.10.0
 huggingface_hub>=0.0.16
 dill>=0.3.4
 base58>=2.1.1
@@ -34,12 +34,8 @@ sacremoses
 # Spacy depends on typer, and typer had a bug. This is how we make sure we get the fixed version of typer.
 typer>=0.4.1
 
-# Indirect dependency of cached-path
-# pyasn1<0.5.0,>=0.4.8
-# pyasn1-modules>=0.2.8
-
 # Protobuf is a dependency of wandb and tensorboard, but they are missing this pin.
-protobuf>=3.12.0,<4.0.0
+protobuf>=3.12.0
 
 # We need this for building the Docker image
 traitlets>5.1.1
@@ -50,4 +46,4 @@ traitlets>5.1.1
 # NOTE: we use a special trailing comment on each line to denote which extras
 # each package is needed by. For example, checklist is needed by the 'checklist' extra
 # that you install with 'pip install allennlp[checklist]'.
-checklist==0.0.11  # needed by: checklist
+checklist>=0.0.11  # needed by: checklist

scripts/check_torch_version.py

@@ -45,16 +45,16 @@ def _get_latest_torch_version() -> Tuple[str, str, str]:
 
 
 def _get_torch_version_upper_limit() -> Tuple[str, str, str]:
-    with open("requirements.txt") as f:
+    with open("constraints.txt") as f:
         for line in f:
             # The torch version line should look like:
-            #   "torch>=X.Y.Z,<X.V.0",
-            if "torch>=" in line:
+            #   "torch<X.V.0",
+            if "torch<" in line:
                 version = tuple(line.split("<")[1].strip().split("."))
                 assert len(version) == 3, f"Bad parsed version '{version}'"
                 break
         else:
-            raise RuntimeError("could not find torch version spec in requirements.txt")
+            raise RuntimeError("could not find torch version spec in constraints.txt")
     return cast(Tuple[str, str, str], version)