Merge pull request #112 from freedomofpress/filename-key-regres

test: regression coverage for #111 (KeyError due to filename), also fix ci on master

Author: kushaldas

.circleci/config.yml

@@ -3,51 +3,47 @@ jobs:
   test-run:
     working_directory: ~/sdclientapi
     docker:
-      - image: circleci/python:3.5.3
-        environment:
-          PIPENV_VENV_IN_PROJECT: true
+      - image: circleci/python:3.7
     steps:
       - checkout
       - run: sudo chown -R circleci:circleci /usr/local/bin
-      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.5/site-packages
+      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.7/site-packages
       - restore_cache:
-          key: deps9-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          key: deps9-{{ .Branch }}-{{ checksum "dev-requirements.txt" }}
       - run:
           name: Install dependencies
           command: |
-            set -e
-            sudo pip install pipenv
-            pipenv install --dev --skip-lock
-            pipenv run pip freeze
+            virtualenv --python=python3 .venv
+            source .venv/bin/activate
+            pip install --require-hashes -r dev-requirements.txt
       - save_cache:
-          key: deps9-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
+          key: deps9-{{ .Branch }}-{{ checksum "dev-requirements.txt" }}
           paths:
             - ".venv"
             - "/usr/local/bin"
-            - "/usr/local/lib/python3.5/site-packages"
+            - "/usr/local/lib/python3.7/site-packages"
       - run:
-          name: Run linters
-          command: pipenv run make lint mypy
-      - run:
-          name: Run tests
-          command: pipenv run make test
-      - run:
-          name: Check for known CVEs
-          command: pipenv check
+          name: Run linter, tests, check for known CVEs
+          command: |
+            source .venv/bin/activate
+            make check
+
   test-against-latest-api:
     working_directory: ~/project
     machine:
       enabled: true
+      image: ubuntu-1604:201903-01
     environment:
       DOCKER_API_VERSION: 1.23
     steps:
       - checkout
       - run:
           name: Install dependencies
           command: |
-            sudo pip install pipenv
-            pipenv install --dev --skip-lock
-            pipenv run pip freeze
+            pyenv global 3.7.0
+            virtualenv --python=python3 .venv
+            source .venv/bin/activate
+            pip install --require-hashes -r dev-requirements.txt
       - run:
           name: Download SecureDrop server code
           command: git clone https://github.com/freedomofpress/securedrop.git
@@ -56,7 +52,7 @@ jobs:
           command: |
             cd securedrop
             NUM_SOURCES=5 make -C securedrop dev
-          background: true  
+          background: true
       - run:  # As suggested in https://discuss.circleci.com/t/prevent-race-conditions-by-waiting-for-services-with-dockerize/11215
           name: Install dockerize
           command: |
@@ -74,7 +70,9 @@ jobs:
           name: Remove VCR cassettes and run tests against latest API
           command: |
             rm data/*.yml  # Removing VCR cassettes
-            pipenv run make test
+            source .venv/bin/activate
+            make test
+
 workflows:
   version: 2
   securedrop_ci:

Makefile

@@ -20,7 +20,22 @@ mypy: ## Run the mypy typechecker
 	@mypy sdclientapi
 
 .PHONY: check
-check: lint mypy test ## Run all checks and tests
+check: lint mypy test safety ## Run all checks and tests
+
+.PHONY: safety
+safety: ## Runs `safety check` to check python dependencies for vulnerabilities
+	pip install --upgrade safety && \
+		for req_file in `find . -type f -name '*requirements.txt'`; do \
+			echo "Checking file $$req_file" \
+			&& safety check --full-report -r $$req_file \
+			&& echo -e '\n' \
+			|| exit 1; \
+		done
+
+.PHONY: update-pip-requirements
+update-pip-requirements: ## Updates all Python requirements files via pip-compile.
+	pip-compile --generate-hashes --output-file dev-requirements.txt requirements.in dev-requirements.in
+	pip-compile --generate-hashes --output-file requirements.txt requirements.in
 
 .PHONY: open-coverage-report
 open-coverage-report: ## Open the coverage report in your browser

README.md

@@ -13,23 +13,19 @@ The SDK is currently used by the [SecureDrop Client](https://github.com/freedomo
 ## Quick Start
 
 ```bash
-pip install -U pipenv
-pipenv sync --dev
-pipenv shell
+virtualenv --python=python3 .venv
+source .venv/bin/activate
+pip install --require-hashes -r dev-requirements.txt
 make test
 ```
 
-This project uses [pipenv](https://docs.pipenv.org) to manage all dependencies.
-This is a Python 3 project. When using ``pipenv`` locally, ensure you used the ``--keep-outdated``
-flag to prevent dependencies from being unnecessarily upgraded during normal development.
-
 We cover all the API calls supported by the SecureDrop Journalist Interface API.
 
 ## Testing
 
 The tests are located in the `tests` directory. This project uses [vcrpy](http://vcrpy.readthedocs.io/en/latest/) to record and then reply the API calls so that
 developers will have repeatable results so that they may work offline. `vcrpy` stores YAML
-recordings of the API calls in the `data` directory. 
+recordings of the API calls in the `data` directory.
 
 To run all the test cases, use the following command.
 
@@ -118,7 +114,7 @@ Please read [CONTRIBUTING.md](https://github.com/freedomofpress/securedrop-sdk/b
 
 # Versioning
 
-We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/freedomofpress/securedrop-sdk/tags). 
+We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/freedomofpress/securedrop-sdk/tags).
 
 # License
 

dev-requirements.in

@@ -0,0 +1,11 @@
+coverage
+flake8
+mypy
+pyotp
+pytest
+pytest-cov
+sphinx
+vcrpy
+pyyaml>=5.1,<6
+pytest-mock
+pip-tools