Merge pull request #99 from ephur/refactor_rebase

more or less complete rewrite of application

Author: ephur

.circleci/config.yml

@@ -9,3 +9,6 @@ __pycache__
 htmlcov
 .pytest_cache
 registry.terraform.io/
+.scannerwork/
+coverage.xml
+reports/*

.gitignore

@@ -1,3 +1,4 @@
 [settings]
-known_third_party = atlassian,boto3,botocore,click,deepdiff,google,hcl2,jinja2,lark,mergedeep,moto,pydantic,pytest,yaml
+known_third_party = atlassian,boto3,botocore,click,google,hcl2,jinja2,lark,moto,packaging,pydantic,pydantic_core,pydantic_settings,pytest,yaml
 profile = black
+skip = ["*/__init__.py"]

.isort.cfg

@@ -0,0 +1,18 @@
+# Changes 0.12.0 -> 0.13.0 (2024-06...)
+
+## Notes
+This version reconsiders many of the core elements of the worker application.
+
+The application now relies on pydantic to define the models for what is consumed from the configuration file and the command line. Previously there was a heinous mix of classes which just had attributes and methods seemingly hung on
+them at random. This was a nightmare to maintain and understand.
+
+- the CopierFactory was reorganized to be more modular
+- providers became a package, with a model used to validation all provider configurations
+- all of the configuration file logic was moved into the `commands` module, the logic to handle managing the cofig is all contained there instead of being spread out many places
+- added tfworker.util.log to handle interaction with the user, via the CLI or via a logger in the future
+- made significant strides towards consolidating all exceptions / error handling
+- made significant strides in only having the primary cli.py and commands/ actually cause the program to terminate
+- validation of options and inputs is now handled centrally in the pydantic models instead of being spread out *everywhere*
+
+
+- ... @TODO find time to update this :)

CHANGELOG.md

@@ -1,21 +1,28 @@
 init:
 	poetry install
 
+init-dev:
+	poetry install --with dev
+
 default: lint test
 
-lint: init
+lint: init-dev
 	poetry run flake8 --ignore E501,W503 tfworker tests
 
-format: init
+format: init-dev
 	poetry run black tfworker tests
-	poetry run seed-isort-config || echo "known_third_party setting changed. Please commit pyproject.toml"
+	@poetry run seed-isort-config || echo "known_third_party setting changed. Please commit pyproject.toml"
 	poetry run isort tfworker tests
 
-test: init
+test: init-dev
 	poetry run pytest -p no:warnings --disable-socket
 	poetry run coverage report --fail-under=60 -m --skip-empty
 
-dep-test: init
+ci-test: init-dev
+	poetry run pytest --disable-socket --junitxml=reports/junit.xml
+	poetry run coverage xml -o reports/coverage.xml
+
+dep-test: init-dev
 	poetry run pytest --disable-socket
 	poetry run coverage report --fail-under=60 -m --skip-empty
 

Makefile

@@ -28,16 +28,18 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.11"
+atlassian-python-api = "^3.41"
 boto3 = "^1.34"
 click = "^8.1"
-jinja2 = "^3.1"
 google-cloud-storage = "^2.17"
+jinja2 = "^3.1"
+mergedeep = "^1.3"
+pydantic = "^2.7"
 python-hcl2 = "^4.3"
 pyyaml = "^6.0"
-mergedeep = "^1.3"
 setuptools = "^70.0"
-atlassian-python-api = "^3.41"
-pydantic = "^2.7"
+pydantic-settings = "^2.3.4"
+packaging = "^24.1"
 
 [tool.poetry.scripts]
 worker = 'tfworker.cli:cli'
@@ -46,24 +48,29 @@ worker = 'tfworker.cli:cli'
 optional = true
 
 [tool.poetry.group.dev.dependencies]
-pytest-timeout = "2.3.1"
-ipython = "^8.24"
-pytest = "^8.2"
 black = "^24.4"
-isort = "^5.13"
-seed-isort-config = "^2.2"
+coverage = "^7.5"
+deepdiff = "^7.0"
 flake8 = "^7.0"
-wheel = "^0.43"
+ipython = "^8.24"
+isort = "^5.13"
+pytest = "^8.2"
+pytest-cov = "^5.0"
 pytest-depends = "^1.0"
+pytest-mock = "^3.14"
 pytest-socket = "^0.7"
-coverage = "^7.5"
-pytest-cov = "^5.0"
-moto = {extras = ["sts","dynamodb", "s3"], version = "^5.0"}
-deepdiff = "^7.0"
+pytest-timeout = "2.3.1"
+seed-isort-config = "^2.2"
 Sphinx = "^7.3"
+wheel = "^0.43"
+moto = {version = "^5.0.10", extras = ["sts", "dynamodb", "s3"]}
+
 
 [tool.pytest.ini_options]
-addopts = "--capture=sys --cov=tfworker --cov-report="
+addopts = "--capture=sys --cov=tfworker --cov-report= -m 'not performance'"
+markers = [
+    "performance: mark a test as a performance test"
+]
 
 [build-system]
 requires = ["poetry>=0.12"]

poetry.lock

@@ -1,13 +1 @@
-# Copyright 2020 Richard Maynard ([email protected])
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.