Move GHE requests to UI server (#369)

* Move GHE requests to UI server

Signed-off-by: Christian Kadner <[email protected]>

* Fix incorrect url matching code

Signed-off-by: Christian Kadner <[email protected]>

Signed-off-by: Christian Kadner <[email protected]>

Author: ckadner

api/examples/catalog_api.py

@@ -25,7 +25,8 @@
 
 catalog_upload_file = "./../../bootstrapper/catalog_upload.json"
 
-IBM_GHE_API_TOKEN = env.get("IBM_GHE_API_TOKEN")
+GHE_API_TOKEN = env.get("GHE_API_TOKEN")
+GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com")
 
 
 def get_swagger_client():
@@ -59,7 +60,7 @@ def upload_catalog_assets(upload_file=catalog_upload_file) -> ApiCatalogUploadRe
             upload_items = json.load(f)
 
         upload_body = ApiCatalogUpload(
-            api_access_tokens=[ApiAccessToken(api_token=IBM_GHE_API_TOKEN, url_host="github.ibm.com")],
+            api_access_tokens=[ApiAccessToken(api_token=GHE_API_TOKEN, url_host=GHE_WEB_URL)],
             components=upload_items.get("components"),
             datasets=upload_items.get("datasets"),
             models=upload_items.get("models"),

api/examples/components_api.py

@@ -364,7 +364,7 @@ def main():
     component = list_components(filter_dict={"name": 'Create Secret - Kubernetes Cluster'})[0]
     generate_code(component.id)
     args = {
-        'token': env.get("IBM_GHE_API_TOKEN"),
+        'token': env.get("GHE_API_TOKEN"),
         'url': 'https://raw.github.ibm.com/user/repo/master/secret.yml',
         'name': 'my-test-credential'
     }

api/examples/notebooks_api.py

@@ -35,7 +35,9 @@
 
 yaml_files = sorted(filter(lambda f: "template" not in f, glob("./../../../katalog/notebook-samples/*.yaml", recursive=True)))
 
-IBM_GHE_API_TOKEN = env.get("IBM_GHE_API_TOKEN")
+GHE_API_TOKEN = env.get("GHE_API_TOKEN")
+GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com")
+GHE_RAW_URL = env.get("GHE_RAW_URL", "raw.github.ibm.com")
 
 
 def get_swagger_client():
@@ -103,8 +105,8 @@ def upload_notebook_templates(yaml_files: [str] = yaml_files) -> [str]:
         with open(yaml_file, "rb") as f:
             yaml_dict = yaml.load(f, Loader=yaml.SafeLoader)
 
-        if "github.ibm.com" in yaml_dict["implementation"]["github"]["source"]:
-            api_token = IBM_GHE_API_TOKEN
+        if GHE_WEB_URL in yaml_dict["implementation"]["github"]["source"]:
+            api_token = GHE_API_TOKEN
         else:
             api_token = None
 
@@ -365,9 +367,9 @@ def download_notebooks_from_github():
 
         download_url = url.replace("/blob", "")\
             .replace("github.com", "raw.githubusercontent.com")\
-            .replace("github.ibm.com", "raw.github.ibm.com")
+            .replace(GHE_WEB_URL, GHE_RAW_URL)
 
-        if "github.ibm.com" in url:
+        if GHE_WEB_URL in url:
             headers = {'Authorization': 'token %s' % env.get("IBM_GHE_API_TOKEN")}
         else:
             headers = {}

api/server/swagger_server/controllers_impl/__init__.py

@@ -18,7 +18,9 @@
 # TODO: move into controllers_impl/util.py
 ###############################################################################
 
-ghe_api_token = env.get("GHE_API_TOKEN")
+GHE_API_TOKEN = env.get("GHE_API_TOKEN")
+GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com")
+GHE_RAW_URL = env.get("GHE_RAW_URL", "raw.github.ibm.com")
 
 
 def get_yaml_file_content_from_uploadfile(uploadfile: FileStorage):
@@ -73,15 +75,15 @@ def download_file_content_from_url(url: str, bearer_token: str = None) -> bytes:
     if bearer_token and "?token=" not in url:
         request_headers.update({"Authorization": f"Bearer {bearer_token}"})
 
-    if "github.ibm.com" in url and "?token=" not in url:
-        if not bearer_token and not ghe_api_token:
+    if GHE_WEB_URL in url and "?token=" not in url:
+        if not bearer_token and not GHE_API_TOKEN:
             raise ApiError(f"Must provide API token to access files on GitHub Enterprise: {url}", 422)
         else:
-            request_headers.update({'Authorization': f'token {bearer_token or ghe_api_token}'})
+            request_headers.update({'Authorization': f'token {bearer_token or GHE_API_TOKEN}'})
 
     try:
         raw_url = url.replace("/blob/", "/") \
-            .replace("/github.ibm.com/", "/raw.github.ibm.com/") \
+            .replace(GHE_WEB_URL, GHE_RAW_URL) \
             .replace("/github.com/", "/raw.githubusercontent.com/")
 
         response = requests.get(raw_url, allow_redirects=True, headers=request_headers)

api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py

@@ -15,7 +15,8 @@
 from werkzeug.datastructures import FileStorage
 
 from swagger_server.controllers_impl import download_file_content_from_url, \
-    get_yaml_file_content_from_uploadfile, validate_id, ghe_api_token
+    get_yaml_file_content_from_uploadfile, validate_id, GHE_API_TOKEN, \
+    GHE_WEB_URL, GHE_RAW_URL
 from swagger_server.data_access.minio_client import store_file, delete_objects, \
     get_file_content_and_url, enable_anonymous_read_access, NoSuchKey, \
     create_tarfile, get_object_url
@@ -454,7 +455,7 @@ def _upload_notebook_yaml(yaml_file_content: AnyStr, name=None, access_token=Non
                    file_name="requirements.txt", file_content=requirements_all.encode())
 
     # if the url included an access token, replace the original url with the s3 url
-    if "?token=" in url or "github.ibm.com" in url:
+    if "?token=" in url or GHE_WEB_URL in url:
         api_notebook.url = s3_url
         update_multiple(ApiNotebook, [notebook_id], "url", s3_url)
         enable_anonymous_read_access(bucket_name="mlpipeline", prefix="notebooks/*")
@@ -467,14 +468,14 @@ def _download_notebook(url: str, enterprise_github_api_token: str) -> dict:
     request_headers = dict()
 
     # TODO: re-use ./init.py#download_file_content_from_url
-    if "github.ibm.com" in url and "?token=" not in url:
-        if not enterprise_github_api_token and not ghe_api_token:
+    if GHE_WEB_URL in url and "?token=" not in url:
+        if not enterprise_github_api_token and not GHE_API_TOKEN:
             raise ApiError(f"Must provide API token to access notebooks on Enterprise GitHub: {url}", 422)
         else:
-            request_headers.update({'Authorization': f'token {enterprise_github_api_token or ghe_api_token}'})
+            request_headers.update({'Authorization': f'token {enterprise_github_api_token or GHE_API_TOKEN}'})
 
     try:
-        raw_url = url.replace("/github.ibm.com/", "/raw.github.ibm.com/")\
+        raw_url = url.replace(GHE_WEB_URL, GHE_RAW_URL)\
                      .replace("/github.com/", "/raw.githubusercontent.com/")\
                      .replace("/blob/", "/")
         response = requests.get(raw_url, allow_redirects=True, headers=request_headers)

dashboard/origin-mlx/Dockerfile

@@ -28,7 +28,8 @@ USER node
 # mark as production build
 ENV NODE_ENV=production
 
-# run build on container startup in order to build in environment variables
+# run `build` at container startup time to render the REACT_APP environment
+# variables into the JavaScript bundle that will run on the client Web browser
 #  - https://create-react-app.dev/docs/adding-custom-environment-variables/
 # TODO: find a better solution, i.e.
 #  - https://www.tutorialworks.com/openshift-deploy-react-app/

dashboard/origin-mlx/README.md

@@ -213,7 +213,7 @@ There are a few environment variables that can be defined that dictate how MLX i
 * `REACT_APP_TTL` - The amount of seconds a cached entry remains valid for (24 hours by default)
 * `REACT_APP_CACHE_INTERVAL` - The minimum amount of time in seconds between two checks on the validity of the cache's 
   contents (24 hours by default)
-* `REACT_APP_GHE_API_TOKEN` - Enterprise GitHub API Token to "read" Markdown files from GitHub Enterprise. Only use when
+* `GHE_API_TOKEN` - Enterprise GitHub API Token to "read" Markdown files from GitHub Enterprise. Only use when
   MLX deployment is behind corporate firewall. The minimal set of permission required for the token are `repo` and 
   `admin:org/read:org` (on a private repository).