Initial commit

Author: Jalkhov

.editorconfig

@@ -0,0 +1,10 @@
+# editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+trim_trailing_whitespace = true

README.md

@@ -1,2 +1,44 @@
-# octodir
-Tool for download directorys from GitHub repository
+<div align="center">
+    <img src="logo.png">
+</div>
+
+Tool to download complete directories from a Github repository.
+
+# Installation
+
+```
+$ git clone https://github.com/Jalkhov/Octodir.git
+$ cd Octodir
+$ pip3 setup.py install
+```
+
+# Use
+
+## From the console
+
+```
+$ octodir
+```
+
+**You will be asked for the following data:**
+
+* **Full folder url:** Direct url to the target folder
+  * **Example:** https://github.com/Jalkhov/octodir/tree/stable/octodir
+* **Output folder**: Absolute path of the output directory
+  * You can enter a dot to download in the current working directory
+
+## In code
+
+```python
+from octodir import Octodir
+
+target = 'https://github.com/Jalkhov/Octodir/tree/stable/octodir'
+folder = '.' # Current working directory
+
+Octo = Octodir(target, folder)
+Octo.dowload_folder()
+```
+
+# Support me <3
+
+<a href="https://www.buymeacoffee.com/Jalkhov" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-red.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" ></a>

logo.png

@@ -0,0 +1,3 @@
+from .octodir_core import Octodir
+
+__version__ = '0.1'

octodir/__init__.py

@@ -0,0 +1,14 @@
+import click
+from octodir import Octodir
+
+
+@click.command()
+@click.option("--folder_url", prompt="Full folder url", help="Full url from the folder to download")
+@click.option("--output_folder", prompt="Output folder", help="Folder where files will be downloaded")
+def octodir_cli(folder_url, output_folder):
+    x = Octodir(folder_url, output_folder)
+    x.dowload_folder()
+
+
+if __name__ == '__main__':
+    octodir_cli()

octodir/octodir_cli.py

@@ -0,0 +1,147 @@
+import json
+import os
+import urllib.request
+
+import requests
+from tqdm import tqdm
+
+
+class repo_info:
+    repo = None
+    target_dir = None
+    branch = None
+
+
+class api_urls:
+    recursive = "https://api.github.com/repos/{}/git/trees/{}?recursive=1"
+    no_recursive = "https://api.github.com/repos/{}/git/trees/{}"
+
+
+class OctodirException(Exception):
+    pass
+
+
+def mkdirs(path):
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+
+class Octodir(object):
+
+    def __init__(self, folder_url, output_folder):
+        super(Octodir, self).__init__()
+        self.folder_url = folder_url
+        self.output_folder = output_folder
+
+        self.repo = None
+        self.target_dir = None
+        self.branch = None
+
+    def __get_raw_url(self, file_path, url):
+        tmp_url = url.replace(
+            'https://api.github.com/repos/',
+            'https://raw.githubusercontent.com/')
+        tmp_url = tmp_url.split('/git/blobs/')[0]
+        tmp_url = tmp_url + '/' + self.branch + '/' + file_path
+
+        return tmp_url
+
+    def __get_repo_tree(self):
+        api = requests.get(
+            api_urls.recursive.format(self.repo, self.branch)).text
+        files = json.loads(api)
+
+        output = []
+        location = dict()
+        for (k, i) in enumerate(files['tree']):
+            # If the target dir is in file path, that file
+            # is inside target folder
+            if self.target_dir in i['path']:
+                if i['type'] == 'blob':
+                    tmp = [i['path']]
+                    tmp += [self.__get_raw_url(tmp[0], i['url'])]
+                    output.append(tmp)
+                else:
+                    location[i['path']] = k
+        files = output
+        location = location
+
+        return (files, location)
+
+    def __scrutinize_url(self, folder_url):
+        try:
+            cutted_url = folder_url.replace('https://github.com/', '')
+            splitted_url = cutted_url.split('/')
+
+            owner = splitted_url[0]
+            repo = splitted_url[1]
+            branch = splitted_url[3]
+
+            target_dir = [item for item in splitted_url[4:]]
+
+            repo_data = repo_info()
+            repo_data.repo = owner + '/' + repo
+            repo_data.branch = branch
+            repo_data.target_dir = "/".join(target_dir)
+
+            return repo_data
+        except IndexError:
+            raise IndexError('Invalid repo url')
+
+    def __api_response(self):
+        repo_data = self.__scrutinize_url(self.folder_url)
+        api = requests.get(api_urls.no_recursive.format(
+            repo_data.repo, repo_data.branch)).text
+        response = json.loads(api)
+
+        return response
+
+    def __check_valid_output(self):
+        if os.path.isdir(self.output_folder):
+            return True
+        else:
+            raise OctodirException('Invalid output directory')
+
+    def __download(self, target_folder='*', recursive=True):
+        data = self.__get_repo_tree()
+        files = data[0]
+        location = data[1]
+
+        # mkdirs(".")
+
+        if target_folder == '*':
+            start = 0
+        else:
+            tmp_target = target_folder.replace('./', '')
+            tmp_target = tmp_target.replace('../', '')
+
+            # Remove "/"
+            tmp_target = (tmp_target if tmp_target[-1] != '/'
+                          else tmp_target[:-1])
+            start = location[target_folder]
+
+        with tqdm(total=len(files), desc="Downloading folder...") as pbar:
+            for i in files[start:]:
+
+                ndir = i[0].replace(
+                    self.target_dir, self.target_dir.split('/')[-1:][0])
+                if recursive or ndir.split(target_folder)[1].count('/') \
+                        <= 1:
+
+                    # Check output dir variable
+                    mkdirs(os.path.join(self.output_folder, os.path.dirname(ndir)))
+                    urllib.request.urlretrieve(
+                        i[1], os.path.join(self.output_folder, ndir))
+                pbar.update(1)
+
+    def dowload_folder(self):
+        check_repo = self.__api_response()
+        if 'message' in check_repo:
+            raise OctodirException(check_repo['message'])
+        else:
+            if self.__check_valid_output() is True:
+                scrutinized_url = self.__scrutinize_url(self.folder_url)
+                self.repo = scrutinized_url.repo
+                self.target_dir = scrutinized_url.target_dir
+                self.branch = scrutinized_url.branch
+                self.__download()

octodir/octodir_core.py

@@ -0,0 +1,3 @@
+tqdm==4.54.0
+requests==2.25.1
+click==8.0.0

requirements.txt

@@ -0,0 +1,52 @@
+import os
+import sys
+
+from octodir import __version__
+from setuptools import setup
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+if sys.argv[-1] == "publish":
+    os.system("python setup.py sdist")
+    os.system("twine upload dist/* --skip-existing")
+    sys.exit()
+
+with open(os.path.join(here, "README.md"), encoding="utf-8") as f:
+    long_description = f.read()
+setup(
+    name="Octodir",
+    version=__version__,
+    description="Tool for download directorys from GitHub repository",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/Jalkhov/octodir",
+    author="Pedro Torcatt",
+    author_email="[email protected]",
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        "Development Status :: 4 - Beta",
+        "Programming Language :: Python :: 3",
+        "Natural Language :: English",
+        "Environment :: Console",
+        "License :: OSI Approved :: MIT License",
+
+    ],
+    keywords="github directory download",
+    packages=["octodir"],
+    include_package_data=True,
+    python_requires=">=3.0",
+    install_requires=open(
+        os.path.join(here, "requirements.txt"), encoding="utf-8"
+    )
+    .read()
+    .split("\n"),
+    project_urls={
+        "Bug Reports": "https://github.com/Jalkhov/octodir/issues",
+        "Source": "https://github.com/Jalkhov/octodir/",
+    },
+    entry_points={"console_scripts": [
+        "octodir=octodir.octodir_cli:octodir_cli"]},
+)