Merge branch 'main' into ir_case_when

Author: shirly121

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ on:
       - dev/docs
     tags:
       - "v*"
-  pull_request_target:
+  pull_request:
     branches:
       - main
       - docs
@@ -26,8 +26,8 @@ jobs:
     - name: Checkout Code
       uses: actions/checkout@v3
       with:
-        repository: ${{github.event.pull_request.head.repo.full_name}}
-        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+        ref: ${{ github.event.pull_request.head.ref }}
         submodules: true
         fetch-depth: 0
 
@@ -49,12 +49,12 @@ jobs:
         node-version: 16
 
     - name: Leave a marker
-      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request_target' }}
+      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' }}
       run: |
         touch ${GITHUB_WORKSPACE}/preview-the-docs.mark
 
     - name: Leave the comment on pull request when started
-      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request_target' }}
+      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }}
       uses: actions-cool/maintain-one-comment@v3
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
@@ -171,15 +171,15 @@ jobs:
         fi
 
     - name: Preview using surge
-      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request_target' }}
+      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }}
       run: |
         npm install -g surge
         surge ./docs/_build/latest/html \
               alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh \
               --token ${{ secrets.SURGE_TOKEN }}
 
     - name: Leave the comment on pull request when succeed
-      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request_target' }}
+      if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }}
       uses: actions-cool/maintain-one-comment@v3
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
@@ -201,7 +201,7 @@ jobs:
         files: "preview-the-docs.mark"
 
     - name: Leave the comment on pull request when failed
-      if: ${{ failure() && steps.check_files.outputs.files_exists == 'true' }}
+      if: ${{ failure() && steps.check_files.outputs.files_exists == 'true' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }}
       uses: actions-cool/maintain-one-comment@v3
       with:
         token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/gss.yml

@@ -149,23 +149,6 @@ jobs:
         docker build -t ${{ env.GSS_IMAGE }}:${SHORT_SHA} \
                 -f .github/workflows/docker/graphscope-store.Dockerfile .
 
-    - name: Prepare the log directory
-      run: |
-        # create the helm installation log directory
-        mkdir -p ${{ github.workspace }}/k8s-ci-helm-installation-logs
-
-        # create the demo fresh of helm installation log directory
-        mkdir -p ${{ github.workspace }}/k8s-ci-demo-fresh-of-helm-installation-logs
-
-        # create the demo script of helm installation with pv log directory
-        mkdir -p ${{ github.workspace }}/k8s-ci-demo-script-of-helm-installation-with-pv-logs
-
-        # create the helm test of helm installation with pv log directory
-        mkdir -p ${{ github.workspace }}/k8s-ci-helm-test-of-helm-installation-with-pv-logs
-
-        # create the demo after restart of helm installation with pv log directory
-        mkdir -p ${{ github.workspace }}/k8s-ci-demo-after-restart-of-helm-installation-with-pv-logs
-
     - name: Setup SSH
       run: |
         ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ''
@@ -263,10 +246,10 @@ jobs:
         MODE: stop
 
     - name: upload the k8s logs to artifact
-      if: ${{ always() }}
+      if: ${{ failure() }}
       uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: k8s-test-logs
-        path: ${{ github.workspace }}/k8s-ci-*-logs
+        path: ${{ github.workspace }}/helm-installation-logs
 

charts/graphscope-store/values.yaml

@@ -25,7 +25,7 @@ image:
   registry: registry.cn-hongkong.aliyuncs.com
   repository: graphscope/graphscope-store
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.22.0"
+  tag: "0.23.0"
   digest: ""
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'

coordinator/gscoordinator/cluster_builder.py

@@ -1,26 +1,30 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import base64
 import json
 import logging
 import os
 
 try:
     from kubernetes import client as kube_client
-    from kubernetes import config as kube_config
-    from kubernetes import watch as kube_watch
-    from kubernetes.client import AppsV1Api
-    from kubernetes.client import CoreV1Api
-    from kubernetes.client.rest import ApiException as K8SApiException
-    from kubernetes.config import ConfigException as K8SConfigException
 except ImportError:
     kube_client = None
-    kube_config = None
-    kube_watch = None
-    AppsV1Api = None
-    CoreV1Api = None
-    K8SApiException = None
-    K8SConfigException = None
-
-from pprint import pprint
 
 from graphscope.deploy.kubernetes.resource_builder import ResourceBuilder
 from graphscope.deploy.kubernetes.utils import get_service_endpoints

docs/images/gsflex-case1.png

@@ -116,7 +116,7 @@ Graph neural networks (GNNs) combines superiority of both graph analytics and ma
 
 ````{dropdown} Prepare data and engine for learning
 
-In our example, we train a Graph Convolutional Network (GCN) model to classify the nodes (papers) into 349 categories, each representing a venue (e.g., pre-print or conference). To accomplish this, we first launch a learning engine and construct a graph with features, following the previous step.
+In our example, we train a supervised GraphSAGE model to classify the nodes (papers) into 349 categories, each representing a venue (e.g., pre-print or conference). To accomplish this, we first launch a learning engine and construct a graph with features, following the previous step.
 
 ```python
 # define the features for learning
@@ -153,8 +153,8 @@ from graphscope.learning.examples import EgoGraphSAGE
 from graphscope.learning.examples import EgoSAGESupervisedDataLoader
 from graphscope.learning.examples.tf.trainer import LocalTrainer
 
-# supervised GCN.
-def train_gcn(graph, node_type, edge_type, class_num, features_num,
+# supervised GraphSAGE.
+def train_sage(graph, node_type, edge_type, class_num, features_num,
               hops_num=2, nbrs_num=[25, 10], epochs=2,
               hidden_dim=256, in_drop_rate=0.5, learning_rate=0.01,
 ):
@@ -195,7 +195,7 @@ def train_gcn(graph, node_type, edge_type, class_num, features_num,
     trainer.train(train_data.iterator, loss, optimizer, epochs=epochs)
     trainer.test(test_data.iterator, test_acc)
 
-train_gcn(lg, node_type="paper", edge_type="cites",
+train_sage(lg, node_type="paper", edge_type="cites",
           class_num=349,  # output dimension
           features_num=130,  # input dimension, 128 + kcore + triangle count
 )
@@ -265,7 +265,89 @@ print(q2.all().result())  # should print [[v[2], v[3], v[0], v[1]]]
 
 
 ## Graph Learning Quick Start
+GNN model training with GraphScope is easy and straightforward. You can use the `graphscope` package to train a GNN model on your local machine. Note that
+`tensorflow` is required to run the following example.
 
-TODO(LiSu):
+````{dropdown} Example: Training GraphSAGE Model in GraphScope
+```python
+
+import graphscope
+from graphscope.dataset import load_ogbn_mag
+
+g = load_ogbn_mag()
+
+# define the features for learning
+paper_features = [f"feat_{i}" for i in range(128)]
+
+# launch a learning engine.
+lg = graphscope.graphlearn(g, nodes=[("paper", paper_features)],
+                  edges=[("paper", "cites", "paper")],
+                  gen_labels=[
+                      ("train", "paper", 100, (0, 75)),
+                      ("val", "paper", 100, (75, 85)),
+                      ("test", "paper", 100, (85, 100))
+                  ])
+
+try:
+    # https://www.tensorflow.org/guide/migrate
+    import tensorflow.compat.v1 as tf
+    tf.disable_v2_behavior()
+except ImportError:
+    import tensorflow as tf
+
+import graphscope.learning
+from graphscope.learning.examples import EgoGraphSAGE
+from graphscope.learning.examples import EgoSAGESupervisedDataLoader
+from graphscope.learning.examples.tf.trainer import LocalTrainer
+
+# supervised GraphSAGE
+def train_sage(graph, node_type, edge_type, class_num, features_num,
+              hops_num=2, nbrs_num=[25, 10], epochs=2,
+              hidden_dim=256, in_drop_rate=0.5, learning_rate=0.01,
+):
+    graphscope.learning.reset_default_tf_graph()
+
+    dimensions = [features_num] + [hidden_dim] * (hops_num - 1) + [class_num]
+    model = EgoGraphSAGE(dimensions, act_func=tf.nn.relu, dropout=in_drop_rate)
+
+    # prepare train dataset
+    train_data = EgoSAGESupervisedDataLoader(
+        graph, graphscope.learning.Mask.TRAIN,
+        node_type=node_type, edge_type=edge_type, nbrs_num=nbrs_num, hops_num=hops_num,
+    )
+    train_embedding = model.forward(train_data.src_ego)
+    train_labels = train_data.src_ego.src.labels
+    loss = tf.reduce_mean(
+        tf.nn.sparse_softmax_cross_entropy_with_logits(
+            labels=train_labels, logits=train_embedding,
+        )
+    )
+    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
+
+    # prepare test dataset
+    test_data = EgoSAGESupervisedDataLoader(
+        graph, graphscope.learning.Mask.TEST,
+        node_type=node_type, edge_type=edge_type, nbrs_num=nbrs_num, hops_num=hops_num,
+    )
+    test_embedding = model.forward(test_data.src_ego)
+    test_labels = test_data.src_ego.src.labels
+    test_indices = tf.math.argmax(test_embedding, 1, output_type=tf.int32)
+    test_acc = tf.div(
+        tf.reduce_sum(tf.cast(tf.math.equal(test_indices, test_labels), tf.float32)),
+        tf.cast(tf.shape(test_labels)[0], tf.float32),
+    )
+
+    # train and test
+    trainer = LocalTrainer()
+    trainer.train(train_data.iterator, loss, optimizer, epochs=epochs)
+    trainer.test(test_data.iterator, test_acc)
+
+train_sage(lg, node_type="paper", edge_type="cites",
+          class_num=349,  # output dimension
+          features_num=128,  # input dimension
+)
+
+```
+````
 
 

docs/images/gsflex-case2.png

@@ -61,7 +61,8 @@ traversalMethod
     | traversalMethod_inE  // inE()[.outV()]
     | traversalMethod_bothE  // bothE()[.otherV()]
     | traversalMethod_limit    // limit()
-    | traversalMethod_valueMap  // valueMap()
+    | traversalMethod_valueMap // valueMap()
+    | traversalMethod_elementMap // elementMap()
     | traversalMethod_order  // order()
     | traversalMethod_select  // select()
     | traversalMethod_dedup   // dedup()
@@ -199,6 +200,12 @@ traversalMethod_valueMap
     : 'valueMap' LPAREN stringLiteralList RPAREN
     ;
 
+// elementMap()
+// elementMap('s1', ...)
+traversalMethod_elementMap
+    : 'elementMap' LPAREN stringLiteralList RPAREN
+    ;
+
 // order()
 // order().by
 traversalMethod_order
@@ -238,12 +245,14 @@ traversalMethod_select
 // by()
 // by("name")
 // by(valueMap())
+// by(elementMap())
 // by(out().count())
 // by(T.label/T.id)
 traversalMethod_selectby
     : 'by' LPAREN RPAREN
     | 'by' LPAREN stringLiteral RPAREN
     | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_valueMap RPAREN
+    | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_elementMap RPAREN
     | 'by' LPAREN nestedTraversal RPAREN
     | 'by' LPAREN traversalToken RPAREN
     ;

docs/images/gsflex-case3.png

@@ -75,7 +75,8 @@ public InterOpCollection build() throws OpArgIllegalException, UnsupportedStepEx
             } else if (Utils.equalClass(step, PropertiesStep.class)
                     || Utils.equalClass(step, PropertyMapStep.class)
                     || Utils.equalClass(step, LabelStep.class)
-                    || Utils.equalClass(step, IdStep.class)) {
+                    || Utils.equalClass(step, IdStep.class)
+                    || Utils.equalClass(step, ElementMapStep.class)) {
                 opList.add(StepTransformFactory.VALUES_STEP.apply(step));
             } else if (Utils.equalClass(step, IsStep.class)) {
                 opList.add(StepTransformFactory.IS_STEP.apply(step));

docs/images/gsflex-case4.png

@@ -279,6 +279,13 @@ public Traversal visitTraversalMethod_valueMap(
                 GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()));
     }
 
+    @Override
+    public Traversal visitTraversalMethod_elementMap(
+            GremlinGSParser.TraversalMethod_elementMapContext ctx) {
+        return graphTraversal.elementMap(
+                GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()));
+    }
+
     @Override
     public Traversal visitTraversalMethod_select(
             GremlinGSParser.TraversalMethod_selectContext ctx) {
@@ -335,6 +342,15 @@ public Traversal visitTraversalMethod_select(
                             nestedVisitor.visitTraversalMethod_valueMap(
                                     byCtx.traversalMethod_valueMap());
                     step.modulateBy(nestedTraversal.asAdmin());
+                } else if (byCtx.traversalMethod_elementMap()
+                        != null) { // select(..).by(elementMap('name'))
+                    TraversalMethodVisitor nestedVisitor =
+                            new TraversalMethodVisitor(
+                                    gvisitor, GremlinAntlrToJava.getTraversalSupplier().get());
+                    Traversal nestedTraversal =
+                            nestedVisitor.visitTraversalMethod_elementMap(
+                                    byCtx.traversalMethod_elementMap());
+                    step.modulateBy(nestedTraversal.asAdmin());
                 } else if (byChildCount == 4
                         && byCtx.nestedTraversal() != null) { // select(..).by(out().count())
                     Traversal nestedTraversal = visitNestedTraversal(byCtx.nestedTraversal());

docs/images/gsflex-overview.png

@@ -1507,6 +1507,14 @@
         test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest",
         method = "g_EX11X",
         reason = "unsupported")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ElementMapTest",
+        method = "g_EX11X_elementMap",
+        reason = "cannot get label and id for out and in vertices")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ElementMapTest",
+        method = "g_V_elementMap",
+        reason = "cannot get label and id for out and in vertices")
 // @Graph.OptOut(
 //        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest",
 //        method = "g_V_valueMapXname_ageX",

docs/overview/getting_started.md

@@ -64,6 +64,7 @@ public class IrGremlinTestSuite extends AbstractGremlinSuite {
                 VertexTest.Traversals.class,
                 UnfoldTest.Traversals.class,
                 ValueMapTest.Traversals.class,
+                ElementMapTest.Traversals.class,
                 GroupTest.Traversals.class,
                 GroupCountTest.Traversals.class,
 
@@ -100,6 +101,7 @@ public class IrGremlinTestSuite extends AbstractGremlinSuite {
                 VertexTest.Traversals.class,
                 UnfoldTest.Traversals.class,
                 ValueMapTest.Traversals.class,
+                ElementMapTest.Traversals.class,
                 GroupTest.Traversals.class,
                 GroupCountTest.Traversals.class,
 

interactive_engine/compiler/src/main/antlr4/GremlinGS.g4

@@ -53,6 +53,7 @@ public static GremlinResultParser analyze(Traversal traversal) {
                     || Utils.equalClass(step, SelectStep.class)
                     || Utils.equalClass(step, PropertiesStep.class)
                     || Utils.equalClass(step, PropertyMapStep.class)
+                    || Utils.equalClass(step, ElementMapStep.class)
                     || Utils.equalClass(step, TraversalMapStep.class)
                     || Utils.equalClass(step, MatchStep.class)
                     || Utils.equalClass(step, ExprStep.class)