Merge pull request #322 from MC-kit/devel

Devel

Author: dvp2015

.github/workflows/release-drafter.yml

@@ -7,6 +7,6 @@ jobs:
   draft_release:
     runs-on: ubuntu-latest
     steps:
-      - uses: release-drafter/release-drafter@v5
+      - uses: release-drafter/release-drafter@v6
         env:
           GITHUB_TOKEN: ${{ secrets.MCKIT_GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -71,21 +71,14 @@ jobs:
 
       - name: Publish package on PyPI
         if: steps.check-version.outputs.tag
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_TOKEN }}
+        run: poetry publish --username=__token__ --password=${{ secrets.PYPI_TOKEN }}
 
       - name: Publish package on TestPyPI
         if: "! steps.check-version.outputs.tag"
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_TOKEN }}
-          repository_url: https://test.pypi.org/legacy/
+        run: poetry publish --username=__token__ --password=${{ secrets.TEST_PYPI_TOKEN }}
 
       - name: Publish the release notes
-        uses: release-drafter/release-drafter@v5
+        uses: release-drafter/release-drafter@v6
         with:
           publish: ${{ steps.check-version.outputs.tag != '' }}
           tag: ${{ steps.check-version.outputs.tag }}

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
 
       - name: Load poetry cache
         id: cached-poetry-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             .cache/pypoetry
@@ -105,7 +105,7 @@ jobs:
 
       - name: Load poetry cache
         id: cached-poetry-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             .cache/pypoetry
@@ -159,7 +159,7 @@ jobs:
 
       - name: Load poetry cache
         id: cached-poetry-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             .cache/pypoetry
@@ -202,7 +202,7 @@ jobs:
 
       - name: Load poetry cache
         id: cached-poetry-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             .cache/pypoetry
@@ -236,4 +236,4 @@ jobs:
           nox --force-color --session=coverage -- xml
 
       - name: Upload coverage report
-        uses: codecov/codecov-action@v3.1.4
+        uses: codecov/codecov-action@v4

.pre-commit-config.yaml

@@ -47,19 +47,20 @@ repos:
     rev: "v1.5.4"
     hooks:
       - id: remove-tabs
+
   # Format doc strings
   - repo: https://github.com/DanielNoord/pydocstringformatter
     rev: "v0.7.3"
     hooks:
       - id: pydocstringformatter
-        exclude: _parser\.py$|_tab\.py
+        exclude: _parser\.py$|_tab\.py|extern
 
   - repo: local
     hooks:
       # Upgrade Python code
       - id: pyupgrade
         name: pyupgrade
-        exclude: \.idea|\.cache|^notebooks
+        exclude: \.idea|\.cache|^notebooks|^extern
         entry: pyupgrade
         args:
           - --py39-plus
@@ -69,7 +70,7 @@ repos:
       # Nicely sort imports
       - id: isort
         name: isort
-        exclude: tools|\.idea|\.cache|^notebooks
+        exclude: tools|\.idea|\.cache|^notebooks|^extern
         entry: isort
         language: system
         types: [ python ]
@@ -80,15 +81,15 @@ repos:
         entry: black
         language: system
         types: [ python ]
-        exclude: ^notebooks
+        exclude: ^notebooks|^extern
 
       # Lint and fix Python code
       - id: ruff
         name: ruff
         entry: ruff --force-exclude --fix --config pyproject.toml
         language: system
         types: [ python ]
-        exclude: ^notebooks
+        exclude: ^notebooks|^extern
 
       # Lint SQL code
       - id: sqlfluff-lint

README.rst

@@ -77,13 +77,59 @@ Examples
 
     jsons = [path1, path2, ...]
     material_ids = {p: get_material_id(p) for p in jsons }
-    case_ids = {c:: get_case_id(p) for p in jsons
+    case_ids = {c: get_case_id(p) for p in jsons }
 
     collector = FullDataCollector()
 
-    for json in jsons:
-        inventory = Inventory.from_json(json)
-        collector.append(inventory, material_id=material_ids[json], case_id=case_ids[json])
+    if sequential_load:
+        for json in jsons:
+            inventory = Inventory.from_json(json)
+            collector.append(inventory, material_id=material_ids[json], case_id=case_ids[json])
+
+    else:  # multithreading is allowed for collector as well
+
+        task_list = ...  # list of tuples[directory, case_id, tasks_sequence]
+        threads = 16  # whatever
+
+        def _find_path(arg) -> tuple[int, int, Path]:
+            _case, path, inventory = arg
+            json_path: Path = (Path(path) / inventory).with_suffix(".json")
+            if not json_path.exists():
+                msg = f"Cannot find file {json_path}"
+                raise FindPathError(msg)
+            try:
+                material_id = int(inventory[_LEN_INVENTORY:])
+                case_str = json_path.parent.parts[-1]
+                case_id = int(case_str[_LEN_CASE:])
+            except (ValueError, IndexError) as x:
+                msg = f"Cannot define material_id and case_id from {json_path}"
+                raise FindPathError(msg) from x
+            if case_id != _case:
+                msg = f"Contradicting values of case_id in case path and database: {case_id} != {_case}"
+                raise FindPathError(msg)
+            return material_id, case_id, json_path
+
+        with futures.ThreadPoolExecutor(max_workers=threads) as executor:
+            mcp_futures = [
+                executor.submit(_find_path, arg)
+                for arg in (
+                    (task_case[0], task_case[1], task)
+                    for task_case in task_list
+                    for task in task_case[2].split(",")
+                    if task.startswith("inventory-")
+                )
+            ]
+
+        mips = [x.result() for x in futures.as_completed(mcp_futures)]
+        mips.sort(key=lambda x: x[0:2])  # sort by material_id, case_id
+
+        def _load_json(arg) -> None:
+            collector, material_id, case_id, json_path = arg
+            collector.append(from_json(json_path.read_text(encoding="utf8")), material_id, case_id)
+
+        with futures.ThreadPoolExecutor(max_workers=threads) as executor:
+            executor.map(_load_json, ((collector, *mip) for mip in mips))
+
 
     collected = collector.get_result()
 

noxfile.py

@@ -190,9 +190,7 @@ def isort(s: Session) -> None:
         "adhoc/*.py",
     ]
     cwd = Path.cwd()
-    files_to_process: list[str] = [
-        str(x) for x in sum((list(cwd.glob(p)) for p in search_patterns), [])
-    ]
+    files_to_process: list[str] = [str(x) for p in search_patterns for x in cwd.glob(p)]
     if files_to_process:
         s.run(
             "poetry",