direct logs to stdout for engine pods

Author: siyuan0322

coordinator/gscoordinator/cluster_builder.py

@@ -163,8 +163,8 @@ def __init__(
 
         self._vineyard_requests = {"cpu": vineyard_cpu, "memory": vineyard_mem}
         self._analytical_requests = {"cpu": engine_cpu, "memory": engine_mem}
-        # Give executor a smaller value, since it doesn't need to load the graph
-        self._executor_requests = {"cpu": "2000m", "memory": engine_mem // 4}
+        # Should give executor a smaller value, since it doesn't need to load the graph
+        self._executor_requests = {"cpu": "2000m", "memory": engine_mem}
         self._learning_requests = {"cpu": "1000m", "memory": "256Mi"}
         self._frontend_requests = {"cpu": "200m", "memory": "512Mi"}
         self._dataset_requests = {"cpu": "200m", "memory": "64Mi"}
@@ -266,19 +266,15 @@ def get_engine_container_helper(
         )
         return container
 
-    def _get_tail_if_exists_cmd(fname: str):
-        return f"""while true; do if [ -e {fname} ];
-                   then tail -f {fname}; fi;
-                   sleep 1; done"""
+    def _get_tail_if_exists_cmd(self, fname: str):
+        return (
+            f"while true; do if [ -e {fname} ]; then tail -f {fname}; fi; sleep 1; done"
+        )
 
     def get_analytical_container(self, volume_mounts, with_java=False):
         name = self.analytical_container_name
         image = self._analytical_image if not with_java else self._analytical_java_image
-        args = [
-            "bash",
-            "-c",
-            self._get_tail_if_exists_cmd("/tmp/grape_engine.INFO")
-        ]
+        args = ["bash", "-c", self._get_tail_if_exists_cmd("/tmp/grape_engine.INFO")]
         container = self.get_engine_container_helper(
             name,
             image,
@@ -305,7 +301,7 @@ def get_interactive_executor_container(self, volume_mounts):
         args = [
             "bash",
             "-c",
-            self._get_tail_if_exists_cmd("/var/log/graphscope/current/executor.log")
+            self._get_tail_if_exists_cmd("/var/log/graphscope/current/executor.log"),
         ]
         container = self.get_engine_container_helper(
             name,
@@ -551,7 +547,7 @@ def get_interactive_frontend_container(self):
         args = [
             "bash",
             "-c",
-            self._get_tail_if_exists_cmd("/var/log/graphscope/current/frontend.log")
+            self._get_tail_if_exists_cmd("/var/log/graphscope/current/frontend.log"),
         ]
         container = kube_client.V1Container(name=name, image=image, args=args)
         container.image_pull_policy = self._image_pull_policy

docs/development/how_to_debug.md

@@ -2,9 +2,37 @@ This document shows how to debugging GraphScope under various conditions.
 
 ### Debugging on local deployment
 
+## Find the logs
+
+Most of the logs will be streamed through the stdout of client, you could control the log level by 
+
+```python
+import graphscope
+graphscope.set_option(show_log=True)
+graphscope.set_option(log_level='DEBUG')  # could also be INFO, ERROR
+```
+
+As you may know, GraphScope is composed of three engines, where the detailed log location of each engine is
+
+- Analytical Engine: `/tmp/grape_engine.INFO`
+- Interactive Engine: Inside `/var/log/graphscope/` or `$HOME/log/graphscope` if GraphScope doesn't have permission of `/var/log`. You may find several folders named with a long number, which is the object id of the graph. There is also a `current` folder links to the log folder of latest created interactive instance.
+- Learning Engine: `graphlearn.INFO` in the current directory.
+
+
 
 ### Debugging on Kubernetes deployment
 
+## Find the logs
+
+In kubernetes environment, besides most of the logs still output to console, you could find detailed logs in each pod's stdout, or files inside each pods.
+
+Note: You could use `kubectl logs <pod>` to inspect the stdout of the pod. Use `kubectl logs <pod> -c <container>` to inspect a specific container inside the pod.
+
+- Coordinator: The stdout of coordinator pod.
+- Analytical Engine: The stdout engine container in the engine pod.
+- Interactive Engine: The stdout of executor container in the engine pod for the executor log. And the stdout of interactive-frontend pod for the frontend. The log files resides in the `/var/log/graphscope` of each container, respectively.
+
+
 ## Commands for Debugging
 
 Here is list with commands usually used for checking the status of the GraphScope deployment on K8s.

docs/frequently_asked_questions.rst

@@ -32,7 +32,8 @@ If you don't find an answer to your question here, feel free to file an `Issues`
 
        graphscope.set_option(show_log=True)
 
-    If you are running GraphScope in k8s, you can use `kubectl describe/logs <https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands>`_ to check the log/status of the cluster. If the disk storage is accessible(on local or via Pods), you may also find logs in `/tmp/gs/runtime/logs`.
+    If you are running GraphScope in k8s, you can use `kubectl describe/logs <https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands>`_ to check the log/status of the pods of GraphScope. 
+    If the disk storage is accessible(on local or via Pods), you may also find logs in `/var/log/graphscope/current` or `$HOME/.local/log/graphscope`.
 
 
 4. Why I find more Pods than expected with command `kubectl get pod`?

interactive_engine/assembly/src/bin/graphscope/giectl

@@ -80,6 +80,10 @@ start_frontend() {
   declare -r pid_dir=${GRAPHSCOPE_RUNTIME}/pid/${object_id}
   mkdir -p ${log_dir} ${config_dir} ${pid_dir}
 
+  # make a "current" link
+  unlink ${GS_LOG}/current || true
+  ln -s ${log_dir} ${GS_LOG}/current
+
   declare java_opt="-server
             -verbose:gc
             -Xloggc:${log_dir}/frontend.gc.log
@@ -149,6 +153,10 @@ start_executor() {
   export LD_LIBRARY_PATH=${GRAPHSCOPE_HOME}/lib:${LD_LIBRARY_PATH}
   export DYLD_LIBRARY_PATH=${GRAPHSCOPE_HOME}/lib:${DYLD_LIBRARY_PATH}
 
+  # make a "current" link
+  unlink ${GS_LOG}/current || true
+  ln -s ${log_dir} ${GS_LOG}/current
+
   # set executor config file
   sed -e "s@GRAPH_NAME@${object_id}@g" \
       -e "s@VINEYARD_OBJECT_ID@${object_id}@g" \
@@ -206,9 +214,6 @@ create_gremlin_instance_on_local() {
   mkdir -p ${GS_LOG}
 
   declare -r log_dir=${GS_LOG}/${object_id}
-  # make a "current" link
-  unlink ${GS_LOG}/current || true
-  ln -s ${log_dir} ${GS_LOG}/current
 
   # Frontend use executor rpc port
   network_servers=""