From f6e15998a94f1e4254e1ce623898a2a0805b8742 Mon Sep 17 00:00:00 2001 From: yecol Date: Tue, 13 Jun 2023 11:36:34 +0800 Subject: [PATCH 1/5] add initial version of command-line tool `gsutil`. --- python/graphscope/gsutil/gsutil.py | 215 +++++++++++++++++++ python/graphscope/gsutil/scripts/__init__.py | 0 python/requirements.txt | 1 + 3 files changed, 216 insertions(+) create mode 100644 python/graphscope/gsutil/gsutil.py create mode 100644 python/graphscope/gsutil/scripts/__init__.py diff --git a/python/graphscope/gsutil/gsutil.py b/python/graphscope/gsutil/gsutil.py new file mode 100644 index 000000000000..db93ef895e33 --- /dev/null +++ b/python/graphscope/gsutil/gsutil.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2023 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import click +import subprocess +import io + + +def run_shell_cmd(cmd, workingdir): + """wrapper function to run a shell command/scripts.""" + click.echo(f"run a shell command on cwd={workingdir}. \ncmd=\"{' '.join(cmd)}\"") + proc = subprocess.Popen(cmd, cwd=workingdir, stdout=subprocess.PIPE) + for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): + print(line.rstrip()) + + +class GSUtil(object): + """GraphScope command-line utility + + This is a context for the utility. + """ + + def __init__(self, home=None, debug=False): + self.home = os.path.abspath(home or ".") + self.debug = debug + + +@click.group() +@click.option( + "--repo-home", + envvar="REPO_HOME", + default=".", + help="GraphScope code repo location.", +) +@click.pass_context +def cli(ctx, repo_home): + ctx.obj = GSUtil(repo_home) + + +@click.command() +@click.pass_obj +def install_deps(): + """Install dependencies for building GraphScope.""" + click.echo("install_deps") + + +@click.command() +@click.argument( + "component", + type=click.Choice( + ["interactive", "analytical", "learning", "coordinator", "client"], + case_sensitive=False, + ), + required=False, +) +@click.option( + "--clean", + is_flag=True, + default=False, + help="Flag indicating whether clean previous build.", +) +@click.option( + "--install", + is_flag=True, + default=False, + help="Flag indicating whether install after built binaries.", +) +@click.option( + "--install-prefix", + type=click.Path(), + default="/opt/graphscope", + show_default=True, + help="Install built binaries to customized location.", +) +@click.option( + "--with-java", + is_flag=True, + default=False, + help="Whether build analytical engine with Java support.", +) +@click.option( + "--storage-type", + type=click.Choice(["experimental", "vineyard"], case_sensitive=False), + help="Make gie with specified storage type.", +) +@click.pass_obj +def make(repo, component, clean, install, install_prefix, storage_type, with_java): + """Build executive binaries of COMPONENT. If not given a specific component, build all. + + \f + TODO: maybe without make? + """ + if clean: + click.secho("Cleaning previous build.", fg="green") + cmd = ["make", "clean"] + run_shell_cmd(cmd, repo.home) + return + click.secho( + f"Before making artifacts, please manually source ENVs from ~/.graphscope_env.", + fg="yellow", + ) + click.secho( + f"Begin the make command, to build components [{component}] of GraphScope, with repo = {repo.home}", + fg="green", + ) + cmd = [] + workingdir = repo.home + if component == "interactive": + click.secho("Building interactive engine.", fg="green") + if storage_type == "experimental": + cmd = ["make", "build", 'QUIET_OPT=""'] + workingdir = os.path.join(repo.home, "interactive_engine", "compiler") + if storage_type == "vineyard": + cmd = [ + "mvn", + "install", + "-DskipTests", + "-Drust.compile.mode=release", + "-P", + "graphscope,graphscope-assembly", + ] + workingdir = os.path.join(repo.home, "interactive_engine") + run_shell_cmd(cmd, workingdir) + cmd = ["tar", "xvzf", "graphscope.tar.gz"] + workingdir = os.path.join( + repo.home, "interactive_engine", "assembly", "target" + ) + click.secho(f"Begin to extract, from {workingdir}.", fg="green") + run_shell_cmd(cmd, workingdir) + click.secho(f"GraphScope interactive engine has been built.", fg="green") + if install == True: + cmd = [ + "make", + "interactive-install", + "INSTALL_PREFIX={}".format(install_prefix), + ] + run_shell_cmd(cmd, repo.home) + click.secho( + f"GraphScope interactive engine has been installed to {install_prefix}.", + fg="green", + ) + + if component == "analytical": + cmd = ["make", "analytical"] + if with_java: + cmd = ["make", "analytical-java"] + run_shell_cmd(cmd, repo.home) + click.secho(f"GraphScope analytical engine has been built.", fg="green") + if install == True: + cmd = [ + "make", + "analytical-install", + "INSTALL_PREFIX={}".format(install_prefix), + ] + run_shell_cmd(cmd, repo.home) + click.secho( + f"GraphScope analytical engine has been installed to {install_prefix}.", + fg="green", + ) + + if component == "client": + cmd = ["make", "client"] + run_shell_cmd(cmd, repo.home) + + if component == "coordinator": + cmd = ["make", "coordinator"] + run_shell_cmd(cmd, repo.home) + + if component == None: + click.secho("Building all components.", fg="green") + cmd = ["make", "all"] + if install == True: + cmd = ["make", "install", "INSTALL_PREFIX={}".format(install_prefix)] + run_shell_cmd(cmd, repo.home) + + +@click.command() +def make_image(): + """Make docker images from source code for deployment. + + \f + TODO: fulfill this. + """ + click.echo("make_image") + + +@click.command() +def test(): + """Trigger tests on built artifacts. + + \f + TODO: fulfill this.""" + click.echo("test") + + +cli.add_command(install_deps) +cli.add_command(make) +cli.add_command(make_image) +cli.add_command(test) diff --git a/python/graphscope/gsutil/scripts/__init__.py b/python/graphscope/gsutil/scripts/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/requirements.txt b/python/requirements.txt index 04bbb677676b..1ea152582ee7 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -20,6 +20,7 @@ ujson;python_version>="3.11" PyYAML rich tqdm +click vineyard>=0.15.0;sys_platform!="win32" vineyard-io>=0.15.0;sys_platform!="win32" From 727df7efcb4952ca9a2aff5c8dc9acb8970e2caf Mon Sep 17 00:00:00 2001 From: yecol Date: Tue, 13 Jun 2023 12:06:15 +0800 Subject: [PATCH 2/5] format with isort. --- python/graphscope/gsutil/gsutil.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/graphscope/gsutil/gsutil.py b/python/graphscope/gsutil/gsutil.py index db93ef895e33..a82e350a844a 100644 --- a/python/graphscope/gsutil/gsutil.py +++ b/python/graphscope/gsutil/gsutil.py @@ -16,10 +16,11 @@ # limitations under the License. # +import io import os -import click import subprocess -import io + +import click def run_shell_cmd(cmd, workingdir): From bfd2d40619f79eb1ed163b785e9e134fd112c923 Mon Sep 17 00:00:00 2001 From: yecol Date: Tue, 13 Jun 2023 14:32:56 +0800 Subject: [PATCH 3/5] update python fix flake8 issues. --- python/graphscope/gsutil/gsutil.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/graphscope/gsutil/gsutil.py b/python/graphscope/gsutil/gsutil.py index a82e350a844a..85cdcc40175a 100644 --- a/python/graphscope/gsutil/gsutil.py +++ b/python/graphscope/gsutil/gsutil.py @@ -113,7 +113,7 @@ def make(repo, component, clean, install, install_prefix, storage_type, with_jav run_shell_cmd(cmd, repo.home) return click.secho( - f"Before making artifacts, please manually source ENVs from ~/.graphscope_env.", + "Before making artifacts, please manually source ENVs from ~/.graphscope_env.", fg="yellow", ) click.secho( @@ -144,8 +144,8 @@ def make(repo, component, clean, install, install_prefix, storage_type, with_jav ) click.secho(f"Begin to extract, from {workingdir}.", fg="green") run_shell_cmd(cmd, workingdir) - click.secho(f"GraphScope interactive engine has been built.", fg="green") - if install == True: + click.secho("GraphScope interactive engine has been built.", fg="green") + if install is True: cmd = [ "make", "interactive-install", @@ -162,8 +162,8 @@ def make(repo, component, clean, install, install_prefix, storage_type, with_jav if with_java: cmd = ["make", "analytical-java"] run_shell_cmd(cmd, repo.home) - click.secho(f"GraphScope analytical engine has been built.", fg="green") - if install == True: + click.secho("GraphScope analytical engine has been built.", fg="green") + if install is True: cmd = [ "make", "analytical-install", @@ -183,10 +183,10 @@ def make(repo, component, clean, install, install_prefix, storage_type, with_jav cmd = ["make", "coordinator"] run_shell_cmd(cmd, repo.home) - if component == None: + if component is None: click.secho("Building all components.", fg="green") cmd = ["make", "all"] - if install == True: + if install is True: cmd = ["make", "install", "INSTALL_PREFIX={}".format(install_prefix)] run_shell_cmd(cmd, repo.home) From 1e1c0d494dc153c5345aac49a29a60b18c0cd16e Mon Sep 17 00:00:00 2001 From: yecol Date: Tue, 13 Jun 2023 17:05:25 +0800 Subject: [PATCH 4/5] revise to gsctl and add a doc. --- docs/utilities/gs.md | 32 ++++++++++++++++++- .../{gsutil/gsutil.py => gsctl/gsctl.py} | 14 ++++---- .../{gsutil => gsctl}/scripts/__init__.py | 0 python/setup.py | 9 ++++-- 4 files changed, 45 insertions(+), 10 deletions(-) rename python/graphscope/{gsutil/gsutil.py => gsctl/gsctl.py} (95%) rename python/graphscope/{gsutil => gsctl}/scripts/__init__.py (100%) diff --git a/docs/utilities/gs.md b/docs/utilities/gs.md index 458c2ee244ef..f5f9034fea7a 100644 --- a/docs/utilities/gs.md +++ b/docs/utilities/gs.md @@ -1,3 +1,33 @@ -# Command-line Utility `gs` +# Command-line Utility `gsctl` +`gsctl` is a command-line utility for GraphScope. It is shipped with `graphscope-client` and provides a set of functionalities to make it easy to use GraphScope. These functionalities include building and testing binaries, managing sessions and resources, and more. + +## Install/Update `gsctl` + +Since it is shipped with python package `graphscope-client`, the `gsctl` command will be available in your terminal after installing GraphScope: +```bash +pip install graphscope-client +``` + +In some cases, such as development on `gsctl`, you may want to build it from source. +To do this, navigate to the directory where the source code is located and run the following command: + +```bash +cd REPO_HOME/python +# If you want to develop gsctl, +# please note the entry point is located on: +# /python/graphscope/gsctl/gsctl.py +pip install --editable . +``` +This will install `gsctl` in an editable mode, which means that any changes you make to the source code will be reflected in the installed version of `gsctl`. + +## Commands + +With `gsctl`, you can do the following things. Always remember to + use `--help` on a command to get more information. + +- `gsctl install-deps`, install dependencies for building GraphScope. +- `gsctl make`, build GraphScope executable binaries and artifacts. +- `gsctl make-image`, build GraphScope docker images. +- `gsctl test`, trigger test suites. diff --git a/python/graphscope/gsutil/gsutil.py b/python/graphscope/gsctl/gsctl.py similarity index 95% rename from python/graphscope/gsutil/gsutil.py rename to python/graphscope/gsctl/gsctl.py index 85cdcc40175a..b4573e6e62c3 100644 --- a/python/graphscope/gsutil/gsutil.py +++ b/python/graphscope/gsctl/gsctl.py @@ -31,14 +31,14 @@ def run_shell_cmd(cmd, workingdir): print(line.rstrip()) -class GSUtil(object): +class GSCtl(object): """GraphScope command-line utility This is a context for the utility. """ - def __init__(self, home=None, debug=False): - self.home = os.path.abspath(home or ".") + def __init__(self, repo_home=None, debug=False): + self.home = os.path.abspath("../") self.debug = debug @@ -46,12 +46,12 @@ def __init__(self, home=None, debug=False): @click.option( "--repo-home", envvar="REPO_HOME", - default=".", + type=click.Path(), help="GraphScope code repo location.", ) @click.pass_context def cli(ctx, repo_home): - ctx.obj = GSUtil(repo_home) + ctx.obj = GSCtl(repo_home) @click.command() @@ -202,11 +202,13 @@ def make_image(): @click.command() -def test(): +@click.pass_obj +def test(repo): """Trigger tests on built artifacts. \f TODO: fulfill this.""" + click.secho(f"repo.home = {repo.home}", fg="green") click.echo("test") diff --git a/python/graphscope/gsutil/scripts/__init__.py b/python/graphscope/gsctl/scripts/__init__.py similarity index 100% rename from python/graphscope/gsutil/scripts/__init__.py rename to python/graphscope/gsctl/scripts/__init__.py diff --git a/python/setup.py b/python/setup.py index 827beca0d973..cfd113af7667 100644 --- a/python/setup.py +++ b/python/setup.py @@ -23,9 +23,7 @@ import sys from distutils.cmd import Command -from setuptools import Extension -from setuptools import find_packages -from setuptools import setup +from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext from setuptools.command.build_py import build_py from setuptools.command.develop import develop @@ -326,6 +324,11 @@ def parse_version(root, **kwargs): "Source": "https://github.com/alibaba/GraphScope", "Tracker": "https://github.com/alibaba/GraphScope/issues", }, + entry_points={ + "console_scripts": [ + "gsctl = graphscope.gsctl.gsctl:cli", + ], + }, ) if os.name == "nt": From 13b0fb98eb185ff8222a3c079c0b53ce43d4b997 Mon Sep 17 00:00:00 2001 From: yecol Date: Tue, 13 Jun 2023 17:15:08 +0800 Subject: [PATCH 5/5] sort import. --- python/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index cfd113af7667..bcfc4e7ba486 100644 --- a/python/setup.py +++ b/python/setup.py @@ -23,7 +23,9 @@ import sys from distutils.cmd import Command -from setuptools import Extension, find_packages, setup +from setuptools import Extension +from setuptools import find_packages +from setuptools import setup from setuptools.command.build_ext import build_ext from setuptools.command.build_py import build_py from setuptools.command.develop import develop