refactor: update gee auth process (#915)

Author: 12rambau

.github/workflows/unit.yml

@@ -9,7 +9,9 @@ on:
 env:
   PLANET_API_CREDENTIALS: ${{ secrets.PLANET_API_CREDENTIALS }}
   PLANET_API_KEY: ${{ secrets.PLANET_API_KEY }}
-  EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }}
+  EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }}
+  EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }}
+  EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }}
 
 jobs:
   lint:

docs/source/tutorials/decorator.rst

@@ -78,6 +78,7 @@ Let's import the required modules. All the decorators are stored in the utils mo
 
 .. code:: python
 
+    import ee
     from time import sleep
     import ipyvuetify as v
     import sepal_ui.sepalwidgets as sw
@@ -142,7 +143,7 @@ It's time to use the decorators in the class methods. For this example, we will
         def request_items(self):
             """Connect to gee and request the root assets id's"""
 
-            folder = ee.data.getAssetRoots()[0]["id"]
+            folder = f"projects/{ee.data._cloud_api_user_project}/assets"
             return [
                 asset["id"]
                 for asset

sepal_ui/aoi/aoi_model.py

@@ -154,7 +154,9 @@ def __init__(
         self.gee = gee
         if gee:
             su.init_ee()
-            self.folder = str(folder) or ee.data.getAssetRoots()[0]["id"]
+            self.folder = (
+                str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/"
+            )
 
         # set default values
         self.set_default(vector, admin, asset)

sepal_ui/message/en/locale.json

@@ -42,7 +42,8 @@
       "custom": "Custom",
       "no_access": "It seems like you do not have access to the input asset or it does not exist.",
       "wrong_type": "The type of the selected asset ({}) does not match authorized asset type ({}).",
-      "placeholder": "users/custom_user/custom_asset"
+      "placeholder": "projects/{project}/assets/asset_name",
+      "no_assets": "No user assets found in: '{}'"
     },
     "load_table": {
       "too_small": "The provided file have less than 3 columns. Please provide a complete point file with at least ID, lattitude and longitude columns."

sepal_ui/scripts/decorator.py

@@ -8,6 +8,7 @@
 ...
 """
 
+import json
 import os
 import warnings
 from functools import wraps
@@ -17,7 +18,6 @@
 from warnings import warn
 
 import ee
-import httplib2
 import ipyvuetify as v
 from deprecated.sphinx import versionadded
 
@@ -34,28 +34,52 @@
 
 
 def init_ee() -> None:
-    """Initialize earth engine according to the environment.
+    r"""Initialize earth engine according using a token.
 
-    It will use the creddential file if the EARTHENGINE_TOKEN env variable exist.
-    Otherwise it use the simple Initialize command (asking the user to register if necessary).
+    THe environment used to run the tests need to have a EARTHENGINE_TOKEN variable.
+    The content of this variable must be the copy of a personal credential file that you can find on your local computer if you already run the earth engine command line tool. See the usage question for a github action example.
+
+    - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials``
+    - Linux: ``/home/USERNAME/.config/earthengine/credentials``
+    - MacOS: ``/Users/USERNAME/.config/earthengine/credentials``
+
+    Note:
+        As all init method of pytest-gee, this method will fallback to a regular ``ee.Initialize()`` if the environment variable is not found e.g. on your local computer.
     """
-    # only do the initialization if the credential are missing
     if not ee.data._credentials:
-        # if the credentials token is asved in the environment use it
-        if "EARTHENGINE_TOKEN" in os.environ:
+        credential_folder_path = Path.home() / ".config" / "earthengine"
+        credential_file_path = credential_folder_path / "credentials"
+
+        if "EARTHENGINE_TOKEN" in os.environ and not credential_file_path.exists():
+
             # write the token to the appropriate folder
             ee_token = os.environ["EARTHENGINE_TOKEN"]
-            credential_folder_path = Path.home() / ".config" / "earthengine"
             credential_folder_path.mkdir(parents=True, exist_ok=True)
-            credential_file_path = credential_folder_path / "credentials"
             credential_file_path.write_text(ee_token)
 
+        # Extract the project name from credentials
+        _credentials = json.loads(credential_file_path.read_text())
+        project_id = _credentials.get("project_id", _credentials.get("project", None))
+
+        if not project_id:
+            raise NameError(
+                "The project name cannot be detected. "
+                "Please set it using `earthengine set_project project_name`."
+            )
+
+        # Check if we are using a google service account
+        if _credentials.get("type") == "service_account":
+            ee_user = _credentials.get("client_email")
+            credentials = ee.ServiceAccountCredentials(
+                ee_user, str(credential_file_path)
+            )
+            ee.Initialize(credentials=credentials)
+            ee.data._cloud_api_user_project = project_id
+            return
+
         # if the user is in local development the authentication should
         # already be available
-        ee.Initialize(http_transport=httplib2.Http())
-        assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root
-
-    return
+        ee.Initialize(project=project_id)
 
 
 ################################################################################

sepal_ui/scripts/gee.py

@@ -95,7 +95,7 @@ def get_assets(folder: Union[str, Path] = "") -> List[dict]:
     """
     # set the folder and init the list
     asset_list = []
-    folder = str(folder) or ee.data.getAssetRoots()[0]["id"]
+    folder = str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/"
 
     def _recursive_get(folder, asset_list):
 
@@ -122,7 +122,7 @@ def is_asset(asset_name: str, folder: Union[str, Path] = "") -> bool:
         true if already in folder
     """
     # get the folder
-    folder = str(folder) or ee.data.getAssetRoots()[0]["id"]
+    folder = str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/"
 
     # get all the assets
     asset_list = get_assets(folder)

sepal_ui/scripts/utils.py

@@ -1,6 +1,7 @@
 """All the helper function of sepal-ui."""
 
 import configparser
+import json
 import math
 import os
 import random
@@ -12,7 +13,6 @@
 from urllib.parse import urlparse
 
 import ee
-import httplib2
 import ipyvuetify as v
 import requests
 import tomli
@@ -127,28 +127,52 @@ def get_file_size(filename: Pathlike) -> str:
 
 
 def init_ee() -> None:
-    """Initialize earth engine according to the environment.
+    r"""Initialize earth engine according using a token.
 
-    It will use the creddential file if the EARTHENGINE_TOKEN env variable exist.
-    Otherwise it use the simple Initialize command (asking the user to register if necessary).
+    THe environment used to run the tests need to have a EARTHENGINE_TOKEN variable.
+    The content of this variable must be the copy of a personal credential file that you can find on your local computer if you already run the earth engine command line tool. See the usage question for a github action example.
+
+    - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials``
+    - Linux: ``/home/USERNAME/.config/earthengine/credentials``
+    - MacOS: ``/Users/USERNAME/.config/earthengine/credentials``
+
+    Note:
+        As all init method of pytest-gee, this method will fallback to a regular ``ee.Initialize()`` if the environment variable is not found e.g. on your local computer.
     """
-    # only do the initialization if the credential are missing
     if not ee.data._credentials:
-        # if the credentials token is asved in the environment use it
-        if "EARTHENGINE_TOKEN" in os.environ:
+        credential_folder_path = Path.home() / ".config" / "earthengine"
+        credential_file_path = credential_folder_path / "credentials"
+
+        if "EARTHENGINE_TOKEN" in os.environ and not credential_file_path.exists():
+
             # write the token to the appropriate folder
             ee_token = os.environ["EARTHENGINE_TOKEN"]
-            credential_folder_path = Path.home() / ".config" / "earthengine"
             credential_folder_path.mkdir(parents=True, exist_ok=True)
-            credential_file_path = credential_folder_path / "credentials"
             credential_file_path.write_text(ee_token)
 
+        # Extract the project name from credentials
+        _credentials = json.loads(credential_file_path.read_text())
+        project_id = _credentials.get("project_id", _credentials.get("project", None))
+
+        if not project_id:
+            raise NameError(
+                "The project name cannot be detected. "
+                "Please set it using `earthengine set_project project_name`."
+            )
+
+        # Check if we are using a google service account
+        if _credentials.get("type") == "service_account":
+            ee_user = _credentials.get("client_email")
+            credentials = ee.ServiceAccountCredentials(
+                ee_user, str(credential_file_path)
+            )
+            ee.Initialize(credentials=credentials)
+            ee.data._cloud_api_user_project = project_id
+            return
+
         # if the user is in local development the authentication should
         # already be available
-        ee.Initialize(http_transport=httplib2.Http())
-        assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root
-
-    return
+        ee.Initialize(project=project_id)
 
 
 def normalize_str(msg: str, folder: bool = True) -> str:

sepal_ui/sepalwidgets/inputs.py

@@ -690,7 +690,9 @@ def __init__(
         self.asset_info = None
 
         # if folder is not set use the root one
-        self.folder = str(folder) or ee.data.getAssetRoots()[0]["id"]
+        self.folder = (
+            str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/"
+        )
         self.types = types
 
         # load the default assets
@@ -699,6 +701,8 @@ def __init__(
         # Validate the input as soon as the object is instantiated
         self.observe(self._validate, "v_model")
 
+        self.observe(self._fill_no_data, "items")
+
         # set the default parameters
         kwargs.setdefault("v_model", None)
         kwargs.setdefault("clearable", True)
@@ -714,10 +718,26 @@ def __init__(
         # load the assets in the combobox
         self._get_items()
 
+        self._fill_no_data({})
+
         # add js behaviours
         self.on_event("click:prepend", self._get_items)
         self.observe(self._get_items, "default_asset")
 
+    def _fill_no_data(self, _: dict) -> None:
+        """Fill the items with a no data message if the items are empty."""
+        # Done in this way because v_slots are not working
+        if not self.items:
+            self.v_model = None
+            self.items = [
+                {
+                    "text": ms.widgets.asset_select.no_assets.format(self.folder),
+                    "disabled": True,
+                }
+            ]
+
+        return
+
     @sd.switch("loading")
     def _validate(self, change: dict) -> None:
         """Validate the selected asset. Throw an error message if is not accessible or not in the type list."""

tests/conftest.py

@@ -21,8 +21,9 @@
 
 try:
     su.init_ee()
-except Exception:
-    pass  # try to init earthengine. use ee.data._credentials to skip
+except Exception as e:
+    raise e
+    # pass  # try to init earthengine. use ee.data._credentials to skip
 
 # -- a component to fake the display in Ipython --------------------------------
 
@@ -123,7 +124,7 @@ def gee_dir(_hash: str) -> Optional[Path]:
         pytest.skip("Eathengine is not connected")
 
     # create a test folder with a hash name
-    root = ee.data.getAssetRoots()[0]["id"]
+    root = f"projects/{ee.data._cloud_api_user_project}/assets/"
     gee_dir = Path(root) / f"sepal-ui-{_hash}"
     ee.data.createAsset({"type": "FOLDER"}, str(gee_dir))
 
@@ -197,17 +198,15 @@ def fake_asset(gee_dir: Path) -> Path:
 
 @pytest.fixture(scope="session")
 def gee_user_dir(gee_dir: Path) -> Path:
-    """Return the path to the gee_dir assets without the project elements.
+    """Return the path to the gee_dir assets.
 
     Args:
         gee_dir: the path to the session defined GEE directory
 
     Returns:
         the path to gee_dir
     """
-    legacy_project = Path("projects/earthengine-legacy/assets")
-
-    return gee_dir.relative_to(legacy_project)
+    return gee_dir
 
 
 @pytest.fixture(scope="session")

tests/test_scripts/test_decorator.py

@@ -1,6 +1,9 @@
 """Test the custom decorators."""
 
+import json
+import os
 import warnings
+from pathlib import Path
 
 import ee
 import ipyvuetify as v
@@ -11,11 +14,58 @@
 from sepal_ui.scripts.warning import SepalWarning
 
 
-@pytest.mark.skipif(not ee.data._credentials, reason="GEE is not set")
 def test_init_ee() -> None:
-    """Check that ee can be initialized from sepal_ui."""
-    # check that no error is raised
-    sd.init_ee()
+    """Test the init_ee_from_token function."""
+    credentials_filepath = Path(ee.oauth.get_credentials_path())
+    existing = False
+
+    try:
+        # Reset credentials to force the initialization
+        # It can be initiated from different imports
+        ee.data._credentials = None
+
+        # Get the credentials path
+
+        # Remove the credentials file if it exists
+        if credentials_filepath.exists():
+            existing = True
+            credentials_filepath.rename(credentials_filepath.with_suffix(".json.bak"))
+
+        # Act: Earthengine token should be created
+        sd.init_ee()
+
+        assert credentials_filepath.exists()
+
+        # read the back up and remove the "project_id" key
+        credentials = json.loads(
+            credentials_filepath.with_suffix(".json.bak").read_text()
+        )
+
+        ## 2. Assert when there's no a project associated
+        # remove the project_id key if it exists
+        ee.data._credentials = None
+        credentials.pop("project_id", None)
+        credentials.pop("project", None)
+        if "EARTHENGINE_PROJECT" in os.environ:
+            del os.environ["EARTHENGINE_PROJECT"]
+
+        # write the new credentials
+        credentials_filepath.write_text(json.dumps(credentials))
+
+        with pytest.raises(NameError) as e:
+            sd.init_ee()
+
+        # Access the exception message via `e.value`
+        error_message = str(e.value)
+        assert "The project name cannot be detected" in error_message
+
+    finally:
+        # restore the file
+        if existing:
+            credentials_filepath.with_suffix(".json.bak").rename(credentials_filepath)
+
+        # check that no error is raised
+        sd.init_ee()
 
     return