chore(service): improve api docs

docs/gensidebar.py

@@ -85,6 +85,7 @@ def write_external(desc, link):
 
     toctree(None, max_depth=3, hidden=True, include_hidden=False)
     write("Renku Client", "introduction", "renku-python")
+    write("Renku Service", "service", "renku-python")
 
     toctree(None, max_depth=1, hidden=True)
     write("Get in touch", "get_in_touch", "renku")

docs/service.rst

@@ -0,0 +1,30 @@
+..
+    Copyright 2017-2021 - Swiss Data Science Center (SDSC)
+    A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+    Eidgenössische Technische Hochschule Zürich (ETHZ).
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+Renku Core Service
+==================
+
+The Renku Core service exposes a functionality similar to the Renku CLI via a
+JSON-RPC API.
+
+
+API Specification
+-----------------
+
+To explore the API documentation and test the current API against a running
+instance of Renku you can use the `Swagger UI on renkulab.io
+<https://renkulab.io/swagger>`_.

renku/cli/service.py

@@ -403,7 +403,8 @@ def all_logs(ctx, follow, output_all, errors):
 @service.command(name="apispec")
 def apispec():
     """Return the api spec."""
-    from renku.service.entrypoint import app, get_apispec
+    from renku.service.entrypoint import app
+    from renku.service.views.apispec import get_apispec
 
     with app.test_request_context():
         click.echo(get_apispec(app).to_yaml())

renku/service/config.py

@@ -22,12 +22,16 @@
 
 import pkg_resources
 
+# TODO: #2100 the git access error should have its own error code
 GIT_ACCESS_DENIED_ERROR_CODE = -32000
 GIT_UNKNOWN_ERROR_CODE = -32001
 
 RENKU_EXCEPTION_ERROR_CODE = -32100
 REDIS_EXCEPTION_ERROR_CODE = -32200
 
+# TODO: #2100 according to the JSON-RPC spec this code is reserved for "method not
+# found" - the invalid headers code should either be a custom code or lumped
+# under invalid params code
 INVALID_HEADERS_ERROR_CODE = -32601
 INVALID_PARAMS_ERROR_CODE = -32602
 INTERNAL_FAILURE_ERROR_CODE = -32603
@@ -66,5 +70,7 @@
 SERVICE_API_BASE_PATH = os.getenv("CORE_SERVICE_API_BASE_PATH", "/")
 # path to the swagger spec
 API_SPEC_URL = SERVICE_PREFIX.lstrip("/") + "/spec.json"
+# URL for fetching the OIDC configuration
+OIDC_URL = os.getenv("OIDC_URL", "/auth/realms/Renku/.well-known/openid-configuration")
 
 LOGGER_CONFIG_FILE = Path(pkg_resources.resource_filename("renku", "service/logging.yaml"))

renku/service/entrypoint.py

@@ -22,9 +22,6 @@
 import uuid
 
 import sentry_sdk
-from apispec import APISpec
-from apispec.ext.marshmallow import MarshmallowPlugin
-from apispec_webframeworks.flask import FlaskPlugin
 from flask import Flask, jsonify, request, url_for
 from jwt import InvalidTokenError
 from sentry_sdk import capture_exception
@@ -33,19 +30,12 @@
 from sentry_sdk.integrations.rq import RqIntegration
 
 from renku.service.cache import cache
-from renku.service.config import (
-    API_VERSION,
-    CACHE_DIR,
-    HTTP_SERVER_ERROR,
-    OPENAPI_VERSION,
-    SERVICE_API_BASE_PATH,
-    SERVICE_NAME,
-    SERVICE_PREFIX,
-)
+from renku.service.config import CACHE_DIR, HTTP_SERVER_ERROR, SERVICE_PREFIX
 from renku.service.logger import service_log
 from renku.service.serializers.headers import JWT_TOKEN_SECRET
 from renku.service.utils.json_encoder import SvcJSONEncoder
 from renku.service.views import error_response
+from renku.service.views.apispec import apispec_blueprint
 from renku.service.views.cache import cache_blueprint
 from renku.service.views.config import config_blueprint
 from renku.service.views.datasets import dataset_blueprint
@@ -84,7 +74,7 @@ def root():
         """Root shows basic service information."""
         import renku
 
-        return jsonify({"service_version": renku.__version__, "spec_url": url_for("openapi")})
+        return jsonify({"service_version": renku.__version__, "spec_url": url_for("apispec.openapi")})
 
     @app.route("/health")
     def health():
@@ -93,11 +83,6 @@ def health():
 
         return "renku repository service version {}\n".format(renku.__version__)
 
-    @app.route(SERVICE_PREFIX.rstrip("/") + "/spec.json")
-    def openapi():
-        """Return the OpenAPI spec for this service."""
-        return jsonify(get_apispec(app).to_dict())
-
     return app
 
 
@@ -110,6 +95,7 @@ def build_routes(app):
     app.register_blueprint(jobs_blueprint)
     app.register_blueprint(templates_blueprint)
     app.register_blueprint(version_blueprint)
+    app.register_blueprint(apispec_blueprint)
 
 
 app = create_app()
@@ -179,28 +165,3 @@ def exceptions(e):
 
     app.logger.handlers.extend(service_log.handlers)
     app.run()
-
-
-def get_apispec(app):
-    """Return the apispec."""
-    spec = APISpec(
-        title=SERVICE_NAME,
-        openapi_version=OPENAPI_VERSION,
-        version=API_VERSION,
-        plugins=[FlaskPlugin(), MarshmallowPlugin()],
-        servers=[{"url": SERVICE_API_BASE_PATH}],
-        components={
-            "securitySchemes": {
-                "oidc": {
-                    "type": "openIdConnect",
-                    "openIdConnectUrl": "/auth/realms/Renku/.well-known/openid-configuration",
-                },
-                "JWT": {"type": "apiKey", "name": "Renku-User", "in": "header"},
-                "gitlab-token": {"type": "apiKey", "name": "Authorization", "in": "header"},
-            }
-        },
-        security=[{"oidc": []}, {"JWT": [], "gitlab-token": []}],
-    )
-    for rule in app.url_map.iter_rules():
-        spec.path(view=app.view_functions[rule.endpoint])
-    return spec

renku/service/views/apispec.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service apispec views."""
+from apispec import APISpec
+from apispec.ext.marshmallow import MarshmallowPlugin
+from apispec_webframeworks.flask import FlaskPlugin
+from flask import Blueprint, current_app, jsonify
+
+from renku.service.config import (
+    API_VERSION,
+    OIDC_URL,
+    OPENAPI_VERSION,
+    SERVICE_API_BASE_PATH,
+    SERVICE_NAME,
+    SERVICE_PREFIX,
+)
+
+apispec_blueprint = Blueprint("apispec", __name__, url_prefix=SERVICE_PREFIX)
+
+# security schemes
+oidc_scheme = {"type": "openIdConnect", "openIdConnectUrl": OIDC_URL}
+jwt_scheme = {"type": "apiKey", "name": "Renku-User", "in": "header"}
+gitlab_token_scheme = {"type": "apiKey", "name": "Authorization", "in": "header"}
+
+TOP_LEVEL_DESCRIPTION = """
+This is the API specification of the renku core service. The API follows the
+[JSON-RPC 2.0](https://www.jsonrpc.org/specification) specifications and mirrors
+the functionality of the renku CLI.
+
+The basic API is low-level and requires that the client handles project
+(repository) state in the service cache by invoking the `cache.project_clone`
+method. This returns a `project_id` that is required for many of the other API
+calls. Note that the `project_id` identifies a combination of `git_url` and
+`ref` - i.e. each combination of `git_url` and `ref` receives a different
+`project_id`.
+
+## Higher-level interface
+
+Some API methods allow the client to defer repository management to the service.
+In these cases, the API documentation will include `project_id` _and_
+`git_url`+`ref` in the spec. Note that for such methods, _either_ `project_id`
+_or_ `git_url` (and optionally `ref`) should be passed in the request body.
+
+## Responses
+
+Following the JSON-RPC 2.0 Specification, the methods all return with HTTP code
+200 and include a [response
+object](https://www.jsonrpc.org/specification#response_object) may contain
+either a `result` or an `error` object. If the call succeeds, the returned
+`result` follows the schema documented in the individual methods. In the case of
+an error, the [`error`
+object](https://www.jsonrpc.org/specification#error_object), contains a code and
+a message describing the nature of the error. In addition to the [standard JSON-RPC
+response codes](https://www.jsonrpc.org/specification#error_object), we define application-specific
+codes:
+
+```
+GIT_ACCESS_DENIED_ERROR_CODE = -32000
+GIT_UNKNOWN_ERROR_CODE = -32001
+
+RENKU_EXCEPTION_ERROR_CODE = -32100
+REDIS_EXCEPTION_ERROR_CODE = -32200
+
+INVALID_HEADERS_ERROR_CODE = -32601
+INVALID_PARAMS_ERROR_CODE = -32602
+INTERNAL_FAILURE_ERROR_CODE = -32603
+
+HTTP_SERVER_ERROR = -32000
+```
+
+"""
+
+spec = APISpec(
+    title=SERVICE_NAME,
+    openapi_version=OPENAPI_VERSION,
+    version=API_VERSION,
+    plugins=[FlaskPlugin(), MarshmallowPlugin()],
+    servers=[{"url": SERVICE_API_BASE_PATH}],
+    security=[{"oidc": []}, {"JWT": [], "gitlab-token": []}],
+    info={"description": TOP_LEVEL_DESCRIPTION},
+)
+
+spec.components.security_scheme("oidc", oidc_scheme)
+spec.components.security_scheme("jwt", jwt_scheme)
+spec.components.security_scheme("gitlab-token", gitlab_token_scheme)
+
+
+@apispec_blueprint.route("/spec.json")
+def openapi():
+    """Return the OpenAPI spec for this service."""
+    return jsonify(get_apispec(current_app).to_dict())
+
+
+def get_apispec(app):
+    """Return the apispec."""
+    for rule in current_app.url_map.iter_rules():
+        spec.path(view=app.view_functions[rule.endpoint])
+    return spec

renku/service/views/cache.py

@@ -75,10 +75,18 @@ def upload_file_view(user_data, cache):
     ---
     post:
       description: Upload a file or archive of files.
+      parameters:
+        - in: query
+          schema: FileUploadRequest
       requestBody:
         content:
-          application/json:
-            schema: FileUploadRequest
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                file:
+                  type: string
+                  format: binary
       responses:
         200:
           description: List of uploaded files.
@@ -192,6 +200,9 @@ def migration_check_project_view(user_data, cache):
     ---
     get:
       description: Retrieve migration information for a project.
+      parameters:
+        - in: query
+          schema: ProjectMigrationCheckRequest
       responses:
         200:
           description: Information about required migrations for the project.

renku/service/views/datasets.py

@@ -52,6 +52,9 @@ def list_datasets_view(user_data, cache):
     ---
     get:
       description: List all datasets in a project.
+      parameters:
+        - in: query
+          schema: DatasetListRequest
       responses:
         200:
           description: Listing of all datasets in a project.