feat: support Github dependencies (#284)

Author: antazoey

setup.py

@@ -83,10 +83,10 @@
         "requests>=2.25.1,<3.0",
         "importlib-metadata",
         "singledispatchmethod ; python_version<'3.8'",
-        "tqdm>=4.62.3,<5.0",
         "IPython>=7.25",
         "pytest>=6.0,<7.0",
         "rich>=10.14,<11",
+        "tqdm>=4.62.3,<5.0",
         "typing-extensions ; python_version<'3.8'",
         "web3[tester]>=5.25.0,<6.0.0",
     ],

src/ape/__init__.py

@@ -14,16 +14,7 @@
 from .managers.networks import NetworkManager as _NetworkManager
 from .managers.project import ProjectManager as _ProjectManager
 from .plugins import PluginManager as _PluginManager
-from .utils import get_package_version
-
-__version__ = get_package_version(__name__)
-
-
-# NOTE: DO NOT OVERWRITE
-_python_version = (
-    f"{_sys.version_info.major}.{_sys.version_info.minor}"
-    f".{_sys.version_info.micro} {_sys.version_info.releaselevel}"
-)
+from .utils import USER_AGENT
 
 # Wiring together the application
 
@@ -35,7 +26,7 @@
     DATA_FOLDER=_Path.home().joinpath(".ape"),
     # NOTE: For all HTTP requests we make
     REQUEST_HEADER={
-        "User-Agent": f"Ape/{__version__} (Python/{_python_version})",
+        "User-Agent": USER_AGENT,
     },
     # What we are considering to be the starting project directory
     PROJECT_FOLDER=_Path.cwd(),

src/ape/api/compiler.py

@@ -1,13 +1,16 @@
 from pathlib import Path
 from typing import List, Set
 
+from ape.api import ConfigItem
 from ape.types import ContractType
 
 from .base import abstractdataclass, abstractmethod
 
 
 @abstractdataclass
 class CompilerAPI:
+    config: ConfigItem
+
     @property
     @abstractmethod
     def name(self) -> str:

src/ape/cli/paramtype.py

@@ -4,6 +4,8 @@
 import click
 from click import Context, Parameter
 
+from ape.utils import get_all_files_in_directory
+
 
 class Path(click.Path):
     """
@@ -25,7 +27,4 @@ def convert(
         self, value: Any, param: Optional["Parameter"], ctx: Optional["Context"]
     ) -> List[PathLibPath]:
         path = super().convert(value, param, ctx)
-        if path.is_dir():
-            return list(path.rglob("*.*"))
-
-        return [path]
+        return get_all_files_in_directory(path)

src/ape/managers/compilers.py

@@ -23,8 +23,8 @@ def registered_compilers(self) -> Dict[str, CompilerAPI]:
         registered_compilers = {}
 
         for plugin_name, (extensions, compiler_class) in self.plugin_manager.register_compiler:
-            # TODO: Add config via ``self.config.get_config(plugin_name)``
-            compiler = compiler_class()
+            config = self.config.get_config(plugin_name)
+            compiler = compiler_class(config=config)
 
             for extension in extensions:
 
@@ -42,7 +42,7 @@ def compile(self, contract_filepaths: List[Path]) -> Dict[str, ContractType]:
         for extension in extensions:
             paths_to_compile = [path for path in contract_filepaths if path.suffix == extension]
             for path in paths_to_compile:
-                logger.info(f"Compiling '{self._get_contract_path(path)}'")
+                logger.info(f"Compiling '{self._get_contract_path(path)}'.")
 
             for contract_type in self.registered_compilers[extension].compile(paths_to_compile):
 

src/ape/managers/config.py

@@ -34,7 +34,7 @@ def __post_init__(self):
         # Top level config items
         self.name = user_config.pop("name", "")
         self.version = user_config.pop("version", "")
-        self.dependencies = user_config.pop("dependencies", [])
+        self.dependencies = user_config.pop("dependencies", {})
 
         for plugin_name, config_class in self.plugin_manager.config_class:
             # NOTE: `dict.pop()` is used for checking if all config was processed

src/ape/managers/project.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Collection, Dict, List, Optional, Union
 
 import requests
 from dataclassy import dataclass
@@ -9,12 +9,24 @@
 from ape.exceptions import ProjectError
 from ape.managers.networks import NetworkManager
 from ape.types import Checksum, Compiler, ContractType, PackageManifest, Source
-from ape.utils import compute_checksum
+from ape.utils import compute_checksum, get_all_files_in_directory, github_client
 
 from .compilers import CompilerManager
 from .config import ConfigManager
 
 
+def _create_source_dict(contracts_paths: Collection[Path]) -> Dict[str, Source]:
+    return {
+        str(source): Source(  # type: ignore
+            checksum=Checksum(  # type: ignore
+                algorithm="md5", hash=compute_checksum(source.read_bytes())
+            ),
+            urls=[],
+        )
+        for source in contracts_paths
+    }
+
+
 @dataclass
 class ProjectManager:
     path: Path
@@ -29,17 +41,57 @@ def __post_init__(self):
             self.path = Path(self.path)
 
         self.dependencies = {
-            manifest.name: manifest
-            for manifest in map(self._extract_manifest, self.config.dependencies)
+            n: self._extract_manifest(n, dep_id) for n, dep_id in self.config.dependencies.items()
         }
 
-    def _extract_manifest(self, manifest_uri: str) -> PackageManifest:
-        manifest_dict = requests.get(manifest_uri).json()
-        # TODO: Handle non-manifest URLs e.g. Ape/Brownie projects, Hardhat/Truffle projects, etc.
-        if "name" not in manifest_dict:
-            raise ProjectError("Dependencies must have a name.")
-
-        return PackageManifest.from_dict(manifest_dict)
+    def _extract_manifest(self, name: str, download_path: str) -> PackageManifest:
+        packages_path = self.config.DATA_FOLDER / "packages"
+        packages_path.mkdir(exist_ok=True, parents=True)
+        target_path = packages_path / name
+        target_path.mkdir(exist_ok=True, parents=True)
+
+        if download_path.startswith("https://") or download_path.startswith("http://"):
+            manifest_file_path = target_path / "manifest.json"
+            if manifest_file_path.exists():
+                manifest_dict = json.loads(manifest_file_path.read_text())
+            else:
+                # Download manifest
+                response = requests.get(download_path)
+                manifest_file_path.write_text(response.text)
+                manifest_dict = response.json()
+
+            if "name" not in manifest_dict:
+                raise ProjectError("Dependencies must have a name.")
+
+            return PackageManifest.from_dict(manifest_dict)
+        else:
+            # Github dependency (format: <org>/<repo>@<version>)
+            try:
+                path, version = download_path.split("@")
+            except ValueError:
+                raise ValueError("Invalid Github ID. Must be given as <org>/<repo>@<version>")
+
+            package_contracts_path = target_path / "contracts"
+            is_cached = len([p for p in target_path.iterdir()]) > 0
+
+            if not is_cached:
+                github_client.download_package(path, version, target_path)
+
+            if not package_contracts_path.exists():
+                raise ProjectError(
+                    "Dependency does not have a support structure. Expecting 'contracts/' path."
+                )
+
+            manifest = PackageManifest()
+            sources = [
+                s
+                for s in get_all_files_in_directory(package_contracts_path)
+                if s.name not in ("package.json", "package-lock.json")
+                and s.suffix in self.compilers.registered_compilers
+            ]
+            manifest.sources = _create_source_dict(sources)
+            manifest.contractTypes = self.compilers.compile(sources)
+            return manifest
 
     def __str__(self) -> str:
         return f'Project("{self.path}")'
@@ -146,8 +198,11 @@ def find_in_dir(dir_path: Path) -> Optional[Path]:
         return find_in_dir(self.contracts_folder)
 
     def load_contracts(
-        self, file_paths: Optional[List[Path]] = None, use_cache: bool = True
+        self, file_paths: Optional[Union[List[Path], Path]] = None, use_cache: bool = True
     ) -> Dict[str, ContractType]:
+        if isinstance(file_paths, Path):
+            file_paths = [file_paths]
+
         # Load a cached or clean manifest (to use for caching)
         manifest = use_cache and self.cached_manifest or PackageManifest()
         cached_sources = manifest.sources or {}
@@ -190,16 +245,7 @@ def file_needs_compiling(source: Path) -> bool:
 
         # Update cached contract types & source code entries in cached manifest
         manifest.contractTypes = contract_types
-        cached_sources = {
-            str(source): Source(  # type: ignore
-                checksum=Checksum(  # type: ignore
-                    algorithm="md5", hash=compute_checksum(source.read_bytes())
-                ),
-                urls=[],
-            )
-            for source in sources
-        }
-        manifest.sources = cached_sources
+        manifest.sources = _create_source_dict(sources)
 
         # NOTE: Cache the updated manifest to disk (so ``self.cached_manifest`` reads next time)
         self.manifest_cachefile.write_text(json.dumps(manifest.to_dict()))