diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index 45aab2a99a38..af7e072110db 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -92,7 +92,8 @@ RUN apt-get update \ automake \ iproute2 \ wireshark-common \ - freeradius + freeradius \ + quilt {% if PTF_ENV_PY_VER == "py3" %} RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ @@ -261,9 +262,14 @@ RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump RUN mkdir -p /var/log/supervisor # Install Python-based GNMI client -RUN git clone https://github.com/lguohan/gnxi.git \ +RUN git clone https://github.com/google/gnxi.git \ && cd gnxi \ - && git checkout 3adf8b9 \ + && git checkout 208acfa85f5b5b8717e14896e9d6ee93cfda9d5f + +COPY gnxi-patches/ gnxi/patches/ + +RUN cd gnxi \ + && quilt push -a \ && cd gnmi_cli_py \ {% if PTF_ENV_PY_VER == "mixed" %} && pip install -r requirements.txt diff --git a/dockers/docker-ptf/gnxi-patches/0001-add-xpath_target-option.patch b/dockers/docker-ptf/gnxi-patches/0001-add-xpath_target-option.patch new file mode 100644 index 000000000000..8d1f86e3b538 --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/0001-add-xpath_target-option.patch @@ -0,0 +1,76 @@ +From d01b36e471f387007680e0f07d8bfe32db8e0269 Mon Sep 17 00:00:00 2001 +From: Guohan Lu +Date: Fri, 3 Jul 2020 09:17:32 +0000 +Subject: [PATCH 1/5] add xpath_target option + +Signed-off-by: Guohan Lu +--- + gnmi_cli_py/py_gnmicli.py | 19 ++++++++++++------- + 1 file changed, 12 insertions(+), 7 deletions(-) + +diff --git a/gnmi_cli_py/py_gnmicli.py b/gnmi_cli_py/py_gnmicli.py +index 7c9e92b..062dee7 100644 +--- a/gnmi_cli_py/py_gnmicli.py ++++ b/gnmi_cli_py/py_gnmicli.py +@@ -126,6 +126,9 @@ def _create_parser(): + required=False, action='store_true') + parser.add_argument('-x', '--xpath', type=str, help='The gNMI path utilized' + 'in the GetRequest or Subscirbe', required=True) ++ parser.add_argument('-xt', '--xpath_target', type=str, help='The gNMI prefix' ++ 'target in the GetRequest or Subscirbe', default=None, ++ required=False) + parser.add_argument('-o', '--host_override', type=str, help='Use this as ' + 'Targets hostname/peername when checking it\'s' + 'certificate CN. You can check the cert with:\nopenssl ' +@@ -258,7 +261,7 @@ def _get_val(json_value): + return val + + +-def _get(stub, paths, username, password): ++def _get(stub, paths, username, password, prefix): + """Create a gNMI GetRequest. + + Args: +@@ -266,16 +269,17 @@ def _get(stub, paths, username, password): + paths: gNMI Path + username: (str) Username used when building the channel. + password: (str) Password used when building the channel. ++ prefix: gNMI Path + + Returns: + a gnmi_pb2.GetResponse object representing a gNMI GetResponse. + """ ++ kwargs = {} + if username: # User/pass supplied for Authentication. +- return stub.Get( +- gnmi_pb2.GetRequest(path=[paths], encoding='JSON_IETF'), +- metadata=[('username', username), ('password', password)]) +- return stub.Get(gnmi_pb2.GetRequest(path=[paths], encoding='JSON_IETF')) +- ++ kwargs = {'metadata': [('username', username), ('password', password)]} ++ return stub.Get( ++ gnmi_pb2.GetRequest(prefix=prefix, path=[paths], encoding='JSON_IETF'), ++ **kwargs) + + def _set(stub, paths, set_type, username, password, json_value): + """Create a gNMI SetRequest. +@@ -368,6 +372,7 @@ def main(): + json_value = args['value'] + private_key = args['private_key'] + xpath = args['xpath'] ++ prefix = gnmi_pb2.Path(target=args['xpath_target']) + host_override = args['host_override'] + user = args['username'] + password = args['password'] +@@ -381,7 +386,7 @@ def main(): + if mode == 'get': + print('Performing GetRequest, encoding=JSON_IETF', 'to', target, + ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _get(stub, paths, user, password) ++ response = _get(stub, paths, user, password, prefix) + print('The GetResponse is below\n' + '-'*25 + '\n') + if form == 'protobuff': + print(response) +-- +2.48.1.windows.1 + diff --git a/dockers/docker-ptf/gnxi-patches/0002-Adding-support-for-subscribe-mode-1.patch b/dockers/docker-ptf/gnxi-patches/0002-Adding-support-for-subscribe-mode-1.patch new file mode 100644 index 000000000000..eca4176d39fd --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/0002-Adding-support-for-subscribe-mode-1.patch @@ -0,0 +1,166 @@ +From 53901aba9ead82be21f1408a601b6266dcf1e3e4 Mon Sep 17 00:00:00 2001 +From: macikgozwa <74217992+macikgozwa@users.noreply.github.com> +Date: Mon, 9 Nov 2020 16:19:24 -0800 +Subject: [PATCH 2/5] Adding support for subscribe mode (#1) + +- Adding support for subscribe mode. The code is mostly based on this patch: https://github.com/google/gnxi/pull/65 +- Adding a new parameter to limit the number of updates, e.g. after a number of streaming updates the client would stop listening. It is convenient for testing purposes. +- Changing the sample interval unit to millisecond. This is also required for testing cases. + +Co-authored-by: Murat Acikgoz +--- + gnmi_cli_py/py_gnmicli.py | 102 +++++++++++++++++++++++++++++++++++--- + 1 file changed, 95 insertions(+), 7 deletions(-) + +diff --git a/gnmi_cli_py/py_gnmicli.py b/gnmi_cli_py/py_gnmicli.py +index 062dee7..7152f13 100644 +--- a/gnmi_cli_py/py_gnmicli.py ++++ b/gnmi_cli_py/py_gnmicli.py +@@ -24,9 +24,7 @@ Current supported gNMI features: + - Auto-loads Target cert from Target if not specified + - User/password based authentication + - Certifificate based authentication +- +-Current unsupported gNMI features: +-- Subscribe ++- Subscribe request + """ + + from __future__ import absolute_import +@@ -40,14 +38,16 @@ import re + import ssl + import sys + import six ++import datetime + try: + import gnmi_pb2 + except ImportError: + print('ERROR: Ensure you\'ve installed dependencies from requirements.txt\n' + 'eg, pip install -r requirements.txt') + import gnmi_pb2_grpc ++import grpc + +-__version__ = '0.4' ++__version__ = '0.5' + + _RE_PATH_COMPONENT = re.compile(r''' + ^ +@@ -143,6 +143,21 @@ def _create_parser(): + required=False, action='store_true') + parser.add_argument('-n', '--notls', help='gRPC insecure mode', + required=False, action='store_true') ++ parser.add_argument('--interval', default=10000, type=int, ++ help='sample interval in millisecond (default: 10000ms)') ++ parser.add_argument('--timeout', type=int, help='subscription' ++ 'duration in seconds (default: none)') ++ parser.add_argument('--heartbeat', default=0, type=int, help='heartbeat interval (default: None)') ++ parser.add_argument('--aggregate', action='store_true', help='allow aggregation') ++ parser.add_argument('--suppress', action='store_true', help='suppress redundant') ++ parser.add_argument('--submode', default=2, type=int, ++ help='subscription mode [0=TARGET_DEFINED, 1=ON_CHANGE, 2=SAMPLE]') ++ parser.add_argument('--update_count', default=0, type=int, help='Max number of streaming updates to receive. 0 means no limit.') ++ parser.add_argument('--subscribe_mode', default=0, type=int, help='[0=STREAM, 1=ONCE, 2=POLL]') ++ parser.add_argument('--encoding', default=0, type=int, help='[0=JSON, 1=BYTES, 2=PROTO, 3=ASCII, 4=JSON_IETF]') ++ parser.add_argument('--qos', default=0, type=int, help='') ++ parser.add_argument('--use_alias', action='store_true', help='use alias') ++ parser.add_argument('--prefix', default='', help='gRPC path prefix (default: none)') + return parser + + +@@ -353,6 +368,79 @@ def _open_certs(**kwargs): + return kwargs + + ++def gen_request(paths, opt, prefix): ++ """Create subscribe request for passed xpath. ++ Args: ++ paths: (str) gNMI path. ++ opt: (dict) Command line argument passed for subscribe reqeust. ++ Returns: ++ gNMI SubscribeRequest object. ++ """ ++ mysubs = [] ++ mysub = gnmi_pb2.Subscription(path=paths, mode=opt["submode"], ++ sample_interval=opt["interval"]*1000000, ++ heartbeat_interval=opt['heartbeat']*1000000, ++ suppress_redundant=opt['suppress']) ++ mysubs.append(mysub) ++ ++ if prefix: ++ myprefix = prefix ++ elif opt["prefix"]: ++ myprefix = _parse_path(_path_names(opt["prefix"])) ++ else: ++ myprefix = None ++ ++ if opt["qos"]: ++ myqos = gnmi_pb2.QOSMarking(marking=opt["qos"]) ++ else: ++ myqos = None ++ mysblist = gnmi_pb2.SubscriptionList(prefix=myprefix, mode=opt['subscribe_mode'], ++ allow_aggregation=opt['aggregate'], encoding=opt['encoding'], ++ subscription=mysubs, use_aliases=opt['use_alias'], qos=myqos) ++ mysubreq = gnmi_pb2.SubscribeRequest(subscribe=mysblist) ++ ++ print('Sending SubscribeRequest\n'+str(mysubreq)) ++ yield mysubreq ++ ++ ++def subscribe_start(stub, options, req_iterator): ++ """ RPC Start for Subscribe reqeust ++ Args: ++ stub: (class) gNMI Stub used to build the secure channel. ++ options: (dict) Command line argument passed for subscribe reqeust. ++ req_iterator: gNMI Subscribe Request from gen_request. ++ Returns: ++ Start Subscribe and printing response of gNMI Subscribe Response. ++ """ ++ metadata = [('username', options['username']), ('password', options['password'])] ++ max_update_count = options["update_count"] ++ try: ++ responses = stub.Subscribe(req_iterator, options['timeout'], metadata=metadata) ++ update_count = 0 ++ for response in responses: ++ print('{0} response received: '.format(datetime.datetime.now())) ++ if response.HasField('sync_response'): ++ print(str(response)) ++ elif response.HasField('error'): ++ print('gNMI Error '+str(response.error.code)+\ ++ ' received\n'+str(response.error.message) + str(response.error)) ++ elif response.HasField('update'): ++ print(response) ++ update_count = update_count+1 ++ else: ++ print('Unknown response received:\n'+str(response)) ++ ++ if max_update_count != 0 and update_count == max_update_count: ++ print("Max update count reached {0}".format(update_count)) ++ break ++ except KeyboardInterrupt: ++ print("Subscribe Session stopped by user.") ++ except grpc.RpcError as x: ++ print("grpc.RpcError received:\n%s" %x) ++ except Exception as err: ++ print(err) ++ ++ + def main(): + argparser = _create_parser() + args = vars(argparser.parse_args()) +@@ -414,9 +502,9 @@ def main(): + response = _set(stub, paths, 'delete', user, password, json_value) + print('The SetRequest response is below\n' + '-'*25 + '\n', response) + elif mode == 'subscribe': +- print('This mode not available in this version') +- sys.exit() ++ request_iterator = gen_request(paths, args, prefix) ++ subscribe_start(stub, args, request_iterator) + + + if __name__ == '__main__': +- main() ++ main() +\ No newline at end of file +-- +2.48.1.windows.1 + diff --git a/dockers/docker-ptf/gnxi-patches/0003-gNMI_client-Add-an-option-to-trigger-memory-spike-on.patch b/dockers/docker-ptf/gnxi-patches/0003-gNMI_client-Add-an-option-to-trigger-memory-spike-on.patch new file mode 100644 index 000000000000..42d893b61937 --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/0003-gNMI_client-Add-an-option-to-trigger-memory-spike-on.patch @@ -0,0 +1,167 @@ +From f2b11e45b16ab13485ae14933f30c18ee6336499 Mon Sep 17 00:00:00 2001 +From: yozhao101 <56170650+yozhao101@users.noreply.github.com> +Date: Thu, 14 Jul 2022 15:02:01 -0700 +Subject: [PATCH 3/5] [gNMI_client] Add an option to trigger memory spike on + gNMI server. (#2) + +What is the motivation of this PR? +This PR aims to trigger memory spike on gNMI server side without explicitly closing the channels (TCP connections) from gNMI client side. + +How did you do it? +I added an option --trigger_mem_spike in this python client script such that the user can specify this option to trigger memory spike. + +How did you test/verify it? +I verify this on the ptf docker container binding to the lab device str-s6000-acs-11. The command line I used is as following: +python /gnxi/gnmi_cli_py/py_gnmicli.py -g -t *.*.*.* -p 50051 -m subscribe -x DOCKER_STATS,TEST_STATS -xt STATE_DB -o "ndastreamingservertest" --trigger_mem_spike + +Signed-off-by: Yong Zhao +--- + gnmi_cli_py/py_gnmicli.py | 105 ++++++++++++++++++++++++-------------- + 1 file changed, 67 insertions(+), 38 deletions(-) + +diff --git a/gnmi_cli_py/py_gnmicli.py b/gnmi_cli_py/py_gnmicli.py +index 7152f13..62d4ec8 100644 +--- a/gnmi_cli_py/py_gnmicli.py ++++ b/gnmi_cli_py/py_gnmicli.py +@@ -57,7 +57,8 @@ _RE_PATH_COMPONENT = re.compile(r''' + (?P.*) # gNMI path value + \])?$ + ''', re.VERBOSE) +- ++INVALID_GNMI_CLIENT_CONNECTION_NUMBER = 1 ++GNMI_SERVER_UNAVAILABLE = 2 + + class Error(Exception): + """Module-level Exception class.""" +@@ -157,6 +158,10 @@ def _create_parser(): + parser.add_argument('--encoding', default=0, type=int, help='[0=JSON, 1=BYTES, 2=PROTO, 3=ASCII, 4=JSON_IETF]') + parser.add_argument('--qos', default=0, type=int, help='') + parser.add_argument('--use_alias', action='store_true', help='use alias') ++ parser.add_argument('--create_connections', type=int, nargs='?', const=1, default=1, ++ help='Creates specific number of TCP connections with gNMI server side. ' ++ 'Default number of TCP connections is 1 and use -1 to create ' ++ 'infinite TCP connections.') + parser.add_argument('--prefix', default='', help='gRPC path prefix (default: none)') + return parser + +@@ -435,8 +440,9 @@ def subscribe_start(stub, options, req_iterator): + break + except KeyboardInterrupt: + print("Subscribe Session stopped by user.") +- except grpc.RpcError as x: +- print("grpc.RpcError received:\n%s" %x) ++ except grpc.RpcError as err: ++ print("Received an exception from server side and error message is: '{}'.".format(err)) ++ raise + except Exception as err: + print(err) + +@@ -465,46 +471,69 @@ def main(): + user = args['username'] + password = args['password'] + form = args['format'] ++ create_connections = args['create_connections'] + paths = _parse_path(_path_names(xpath)) + kwargs = {'root_cert': root_cert, 'cert_chain': cert_chain, + 'private_key': private_key} + certs = _open_certs(**kwargs) + creds = _build_creds(target, port, get_cert, certs, notls) +- stub = _create_stub(creds, target, port, host_override) +- if mode == 'get': +- print('Performing GetRequest, encoding=JSON_IETF', 'to', target, +- ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _get(stub, paths, user, password, prefix) +- print('The GetResponse is below\n' + '-'*25 + '\n') +- if form == 'protobuff': +- print(response) +- elif response.notification[0].update[0].val.json_ietf_val: +- print(json.dumps(json.loads(response.notification[0].update[0].val. +- json_ietf_val), indent=2)) +- elif response.notification[0].update[0].val.string_val: +- print(response.notification[0].update[0].val.string_val) +- else: +- print('JSON Format specified, but gNMI Response was not json_ietf_val') +- print(response) +- elif mode == 'set-update': +- print('Performing SetRequest Update, encoding=JSON_IETF', ' to ', target, +- ' with the following gNMI Path\n', '-'*25, '\n', paths, json_value) +- response = _set(stub, paths, 'update', user, password, json_value) +- print('The SetRequest response is below\n' + '-'*25 + '\n', response) +- elif mode == 'set-replace': +- print('Performing SetRequest Replace, encoding=JSON_IETF', ' to ', target, +- ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _set(stub, paths, 'replace', user, password, json_value) +- print('The SetRequest response is below\n' + '-'*25 + '\n', response) +- elif mode == 'set-delete': +- print('Performing SetRequest Delete, encoding=JSON_IETF', ' to ', target, +- ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _set(stub, paths, 'delete', user, password, json_value) +- print('The SetRequest response is below\n' + '-'*25 + '\n', response) +- elif mode == 'subscribe': +- request_iterator = gen_request(paths, args, prefix) +- subscribe_start(stub, args, request_iterator) ++ ++ if create_connections < -1: ++ print(''' ++ Default number of TCP connections with gNMI server is 1. ++ Please use the '--create_connections ' to ++ create TCP connections or use '--create_connections -1' to ++ create infinite TCP connections. ++ ''', file=sys.stderr) ++ sys.exit(INVALID_GNMI_CLIENT_CONNECTION_NUMBER) ++ ++ while True: ++ if create_connections > 0: ++ create_connections -= 1 ++ elif create_connections == 0: ++ break ++ ++ try: ++ stub = _create_stub(creds, target, port, host_override) ++ if mode == 'get': ++ print('Performing GetRequest, encoding=JSON_IETF', 'to', target, ++ ' with the following gNMI Path\n', '-'*25, '\n', paths) ++ response = _get(stub, paths, user, password, prefix) ++ print('The GetResponse is below\n' + '-'*25 + '\n') ++ if form == 'protobuff': ++ print(response) ++ elif response.notification[0].update[0].val.json_ietf_val: ++ print(json.dumps(json.loads(response.notification[0].update[0].val. ++ json_ietf_val), indent=2)) ++ elif response.notification[0].update[0].val.string_val: ++ print(response.notification[0].update[0].val.string_val) ++ else: ++ print('JSON Format specified, but gNMI Response was not json_ietf_val') ++ print(response) ++ elif mode == 'set-update': ++ print('Performing SetRequest Update, encoding=JSON_IETF', ' to ', target, ++ ' with the following gNMI Path\n', '-'*25, '\n', paths, json_value) ++ response = _set(stub, paths, 'update', user, password, json_value) ++ print('The SetRequest response is below\n' + '-'*25 + '\n', response) ++ elif mode == 'set-replace': ++ print('Performing SetRequest Replace, encoding=JSON_IETF', ' to ', target, ++ ' with the following gNMI Path\n', '-'*25, '\n', paths) ++ response = _set(stub, paths, 'replace', user, password, json_value) ++ print('The SetRequest response is below\n' + '-'*25 + '\n', response) ++ elif mode == 'set-delete': ++ print('Performing SetRequest Delete, encoding=JSON_IETF', ' to ', target, ++ ' with the following gNMI Path\n', '-'*25, '\n', paths) ++ response = _set(stub, paths, 'delete', user, password, json_value) ++ print('The SetRequest response is below\n' + '-'*25 + '\n', response) ++ elif mode == 'subscribe': ++ request_iterator = gen_request(paths, args, prefix) ++ subscribe_start(stub, args, request_iterator) ++ except grpc.RpcError as err: ++ if err.code() == grpc.StatusCode.UNAVAILABLE: ++ print("Client receives an exception '{}' indicating gNMI server is shut down and Exiting ..." ++ .format(err.details())) ++ sys.exit(GNMI_SERVER_UNAVAILABLE) + + + if __name__ == '__main__': +- main() +\ No newline at end of file ++ main() +-- +2.48.1.windows.1 + diff --git a/dockers/docker-ptf/gnxi-patches/0004-Add-support-for-streaming-structured-events-in-night.patch b/dockers/docker-ptf/gnxi-patches/0004-Add-support-for-streaming-structured-events-in-night.patch new file mode 100644 index 000000000000..8d82b86492d6 --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/0004-Add-support-for-streaming-structured-events-in-night.patch @@ -0,0 +1,88 @@ +From b85e4ab565df4472fdcfcded2a53e6bf57aa493f Mon Sep 17 00:00:00 2001 +From: Zain Budhwani <99770260+zbud-msft@users.noreply.github.com> +Date: Tue, 20 Jun 2023 13:58:42 -0700 +Subject: [PATCH 4/5] Add support for streaming structured events in nightly + test (#3) + +What is the motivation of this PR? + +This PR adds support for testing structured events as part of nightly test, by adding params: filter_event and event_op_file +When subscribing to EVENTS/all we will check for filter_event as part of response and then we will add response that contains filter to op_file. Test will then fetch this output file then do parsing logic for yang validation. + +How did you do it? + +I added options filter_event and event_op_file for supporting nightly tests for event + +How did you test/verify it? + +I verify this on the ptf docker container binding to the lab device str-s6000-on-6. The command line I used is as following: +python /gnxi/gnmi_cli_py/py_gnmicli.py -g -t ... -p 50051 -m subscribe -x all -xt EVENTS -o "ndastreamingservertest" --filter_event_regex sonic-events-bgp:bgp-state +--- + gnmi_cli_py/py_gnmicli.py | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +diff --git a/gnmi_cli_py/py_gnmicli.py b/gnmi_cli_py/py_gnmicli.py +index 62d4ec8..c46592a 100644 +--- a/gnmi_cli_py/py_gnmicli.py ++++ b/gnmi_cli_py/py_gnmicli.py +@@ -37,6 +37,7 @@ import os + import re + import ssl + import sys ++import string + import six + import datetime + try: +@@ -162,6 +163,7 @@ def _create_parser(): + help='Creates specific number of TCP connections with gNMI server side. ' + 'Default number of TCP connections is 1 and use -1 to create ' + 'infinite TCP connections.') ++ parser.add_argument('--filter_event_regex', help='Regex to filter event when querying events path') + parser.add_argument('--prefix', default='', help='gRPC path prefix (default: none)') + return parser + +@@ -408,6 +410,12 @@ def gen_request(paths, opt, prefix): + yield mysubreq + + ++def check_event_response(response, filter_event_regex): ++ resp = str(response) ++ match = re.findall(filter_event_regex, resp) ++ return match ++ ++ + def subscribe_start(stub, options, req_iterator): + """ RPC Start for Subscribe reqeust + Args: +@@ -419,6 +427,8 @@ def subscribe_start(stub, options, req_iterator): + """ + metadata = [('username', options['username']), ('password', options['password'])] + max_update_count = options["update_count"] ++ filter_event_regex = options["filter_event_regex"] ++ + try: + responses = stub.Subscribe(req_iterator, options['timeout'], metadata=metadata) + update_count = 0 +@@ -430,8 +440,17 @@ def subscribe_start(stub, options, req_iterator): + print('gNMI Error '+str(response.error.code)+\ + ' received\n'+str(response.error.message) + str(response.error)) + elif response.HasField('update'): +- print(response) +- update_count = update_count+1 ++ if filter_event_regex is not None: ++ if filter_event_regex is not "": ++ match = check_event_response(response, filter_event_regex) ++ if len(match) is not 0: ++ print(response) ++ update_count = update_count + 1 ++ else: ++ raise Exception("Filter event regex should not be empty") ++ else: ++ print(response) ++ update_count = update_count+1 + else: + print('Unknown response received:\n'+str(response)) + +-- +2.48.1.windows.1 + diff --git a/dockers/docker-ptf/gnxi-patches/0005-Enhance-gnmi_cli_py-4.patch b/dockers/docker-ptf/gnxi-patches/0005-Enhance-gnmi_cli_py-4.patch new file mode 100644 index 000000000000..f9fc30d96eb0 --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/0005-Enhance-gnmi_cli_py-4.patch @@ -0,0 +1,323 @@ +From 3adf8b97755b49947e465b5a14645f11e79fa0cd Mon Sep 17 00:00:00 2001 +From: ganglv <88995770+ganglyu@users.noreply.github.com> +Date: Fri, 8 Sep 2023 13:15:29 +0800 +Subject: [PATCH 5/5] Enhance gnmi_cli_py (#4) + +1. Upgrade grpcio and grpcio-tools +2. Support origin in gnmi prefix +3. Print grpc error code +4. Support PROTO encoding for gnmi get +5. Ignore / in [] +6. Support proto_bytes for gnmi set +7. Support multiple path and value for gnmi set, get and subscribe +--- + gnmi_cli_py/py_gnmicli.py | 161 ++++++++++++++++++++++++++--------- + gnmi_cli_py/requirements.txt | 4 +- + 2 files changed, 125 insertions(+), 40 deletions(-) + +diff --git a/gnmi_cli_py/py_gnmicli.py b/gnmi_cli_py/py_gnmicli.py +index c46592a..0ea6f3d 100644 +--- a/gnmi_cli_py/py_gnmicli.py ++++ b/gnmi_cli_py/py_gnmicli.py +@@ -111,8 +111,11 @@ def _create_parser(): + 'file (prepend filename with "@")', default='get') + parser.add_argument('-val', '--value', type=str, help='Value for SetRequest.' + '\nCan be Leaf value or JSON file. If JSON file, prepend' +- ' with "@"; eg "@interfaces.json".', +- required=False) ++ ' with "@"; eg "@interfaces.json".' ++ '\n If empty value for delete operation, use "".', ++ nargs="+", required=False) ++ parser.add_argument('--proto', type=str, help='Output files for proto bytes', ++ nargs="*", required=False) + parser.add_argument('-pkey', '--private_key', type=str, help='Fully' + 'quallified path to Private key to use when establishing' + 'a gNMI Channel to the Target', required=False) +@@ -127,10 +130,13 @@ def _create_parser(): + 'Target when establishing secure gRPC channel.', + required=False, action='store_true') + parser.add_argument('-x', '--xpath', type=str, help='The gNMI path utilized' +- 'in the GetRequest or Subscirbe', required=True) ++ 'in the GetRequest or Subscirbe', nargs="+", required=True) + parser.add_argument('-xt', '--xpath_target', type=str, help='The gNMI prefix' + 'target in the GetRequest or Subscirbe', default=None, + required=False) ++ parser.add_argument('-xo', '--xpath_origin', type=str, help='The gNMI prefix' ++ 'origin in the GetRequest, SetRequest or Subscirbe', default=None, ++ required=False) + parser.add_argument('-o', '--host_override', type=str, help='Use this as ' + 'Targets hostname/peername when checking it\'s' + 'certificate CN. You can check the cert with:\nopenssl ' +@@ -181,7 +187,53 @@ def _path_names(xpath): + """ + if not xpath or xpath == '/': # A blank xpath was provided at CLI. + return [] +- return xpath.strip().strip('/').split('/') # Remove leading and trailing '/'. ++ xpath = xpath.strip().strip('/') ++ path = [] ++ xpath = xpath + '/' ++ # insideBrackets is true when at least one '[' has been found and no ++ # ']' has been found. It is false when a closing ']' has been found. ++ insideBrackets = False ++ # begin marks the beginning of a path element, which is separated by ++ # '/' unclosed between '[' and ']'. ++ begin = 0 ++ # end marks the end of a path element, which is separated by '/' ++ # unclosed between '[' and ']'. ++ end = 0 ++ ++ # Split the given string using unescaped '/'. ++ while end < len(xpath): ++ if xpath[end] == '/': ++ if not insideBrackets: ++ # Current '/' is a valid path element ++ # separator. ++ if end > begin: ++ path.append(xpath[begin:end]) ++ end += 1 ++ begin = end ++ else: ++ # Current '/' must be part of a List key value ++ # string. ++ end += 1 ++ elif xpath[end] == '[': ++ if (end == 0 or xpath[end-1] != '\\') and not insideBrackets: ++ # Current '[' is unescacped, and is the ++ # beginning of List key-value pair(s) string. ++ insideBrackets = True ++ end += 1 ++ elif xpath[end] == ']': ++ if (end == 0 or xpath[end-1] != '\\') and insideBrackets: ++ # Current ']' is unescacped, and is the end of ++ # List key-value pair(s) string. ++ insideBrackets = False ++ end += 1 ++ else: ++ end += 1 ++ ++ if insideBrackets: ++ print("missing ] in path string: %s" % xpath) ++ return [] ++ ++ return path + + + def _parse_path(p_names): +@@ -275,6 +327,16 @@ def _get_val(json_value): + raise JsonReadError('Error while loading JSON: %s' % str(e)) + val.json_ietf_val = json.dumps(set_json).encode() + return val ++ elif '$' in json_value: ++ try: ++ proto_bytes = six.moves.builtins.open(json_value.strip('$'), 'rb').read() ++ except (IOError, ValueError) as e: ++ raise ValueError('Error while loading %s: %s' % (json_value.strip('$'), str(e))) ++ val.proto_bytes = proto_bytes ++ return val ++ elif json_value == '': ++ # GNMI client should use delete operation for empty string ++ return None + coerced_val = _format_type(json_value) + type_to_value = {bool: 'bool_val', int: 'int_val', float: 'float_val', + str: 'string_val'} +@@ -283,16 +345,16 @@ def _get_val(json_value): + return val + + +-def _get(stub, paths, username, password, prefix): ++def _get(stub, paths, username, password, prefix, encoding): + """Create a gNMI GetRequest. + + Args: + stub: (class) gNMI Stub used to build the secure channel. +- paths: gNMI Path ++ paths: (list) gNMI Path + username: (str) Username used when building the channel. + password: (str) Password used when building the channel. + prefix: gNMI Path +- ++ encoding: (int) Encoding + Returns: + a gnmi_pb2.GetResponse object representing a gNMI GetResponse. + """ +@@ -300,35 +362,36 @@ def _get(stub, paths, username, password, prefix): + if username: # User/pass supplied for Authentication. + kwargs = {'metadata': [('username', username), ('password', password)]} + return stub.Get( +- gnmi_pb2.GetRequest(prefix=prefix, path=[paths], encoding='JSON_IETF'), ++ gnmi_pb2.GetRequest(prefix=prefix, path=paths, encoding=encoding), + **kwargs) + +-def _set(stub, paths, set_type, username, password, json_value): ++def _set(stub, prefix, paths, set_type, username, password, value_list): + """Create a gNMI SetRequest. + + Args: + stub: (class) gNMI Stub used to build the secure channel. +- paths: gNMI Path ++ paths: (list) gNMI Path + set_type: (str) Type of gNMI SetRequest. + username: (str) Username used when building the channel. + password: (str) Password used when building the channel. +- json_value: (str) JSON_IETF or file. ++ value_list: (list) JSON_IETF or file. + + Returns: + a gnmi_pb2.SetResponse object representing a gNMI SetResponse. + """ +- if json_value: # Specifying ONLY a path is possible (eg delete). +- val = _get_val(json_value) +- path_val = gnmi_pb2.Update(path=paths, val=val,) +- ++ delete_list = [] ++ update_list = [] ++ for path, value in zip(paths, value_list): ++ val = _get_val(value) ++ if val is None: ++ delete_list.append(path) ++ else: ++ path_val = gnmi_pb2.Update(path=path, val=val,) ++ update_list.append(path_val) + kwargs = {} + if username: + kwargs = {'metadata': [('username', username), ('password', password)]} +- if set_type == 'delete': +- return stub.Set(gnmi_pb2.SetRequest(delete=[paths]), **kwargs) +- elif set_type == 'update': +- return stub.Set(gnmi_pb2.SetRequest(update=[path_val]), **kwargs) +- return stub.Set(gnmi_pb2.SetRequest(replace=[path_val]), **kwargs) ++ return stub.Set(gnmi_pb2.SetRequest(prefix=prefix, delete=delete_list, update=update_list), **kwargs) + + + def _build_creds(target, port, get_cert, certs, notls): +@@ -378,17 +441,18 @@ def _open_certs(**kwargs): + def gen_request(paths, opt, prefix): + """Create subscribe request for passed xpath. + Args: +- paths: (str) gNMI path. ++ paths: (list) gNMI path. + opt: (dict) Command line argument passed for subscribe reqeust. + Returns: + gNMI SubscribeRequest object. + """ + mysubs = [] +- mysub = gnmi_pb2.Subscription(path=paths, mode=opt["submode"], +- sample_interval=opt["interval"]*1000000, +- heartbeat_interval=opt['heartbeat']*1000000, +- suppress_redundant=opt['suppress']) +- mysubs.append(mysub) ++ for path in paths: ++ mysub = gnmi_pb2.Subscription(path=path, mode=opt["submode"], ++ sample_interval=opt["interval"]*1000000, ++ heartbeat_interval=opt['heartbeat']*1000000, ++ suppress_redundant=opt['suppress']) ++ mysubs.append(mysub) + + if prefix: + myprefix = prefix +@@ -482,16 +546,22 @@ def main(): + get_cert = args['get_cert'] + root_cert = args['root_cert'] + cert_chain = args['cert_chain'] +- json_value = args['value'] ++ value_list = args['value'] + private_key = args['private_key'] +- xpath = args['xpath'] +- prefix = gnmi_pb2.Path(target=args['xpath_target']) ++ xpath_list = args['xpath'] ++ proto_list = args['proto'] ++ # In the case that a prefix is specified, it MUST specify any required origin ++ prefix = gnmi_pb2.Path(origin=args['xpath_origin'], target=args['xpath_target']) + host_override = args['host_override'] + user = args['username'] + password = args['password'] + form = args['format'] + create_connections = args['create_connections'] +- paths = _parse_path(_path_names(xpath)) ++ encoding = args['encoding'] ++ paths = [] ++ if xpath_list: ++ for xpath in xpath_list: ++ paths.append(_parse_path(_path_names(xpath))) + kwargs = {'root_cert': root_cert, 'cert_chain': cert_chain, + 'private_key': private_key} + certs = _open_certs(**kwargs) +@@ -517,13 +587,25 @@ def main(): + if mode == 'get': + print('Performing GetRequest, encoding=JSON_IETF', 'to', target, + ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _get(stub, paths, user, password, prefix) ++ response = _get(stub, paths, user, password, prefix, encoding) + print('The GetResponse is below\n' + '-'*25 + '\n') +- if form == 'protobuff': ++ if encoding == 2: ++ i = 0 ++ for notification in response.notification: ++ for update in notification.update: ++ if i >= len(proto_list): ++ print("Not enough files: %s" % str(proto_list)) ++ sys.exit(1) ++ with open(proto_list[i], 'wb') as fp: ++ fp.write(update.val.proto_bytes) ++ i += 1 ++ elif form == 'protobuff': + print(response) + elif response.notification[0].update[0].val.json_ietf_val: +- print(json.dumps(json.loads(response.notification[0].update[0].val. +- json_ietf_val), indent=2)) ++ for notification in response.notification: ++ for update in notification.update: ++ print(json.dumps(json.loads(update.val.json_ietf_val), indent=2)) ++ print('-'*25 + '\n') + elif response.notification[0].update[0].val.string_val: + print(response.notification[0].update[0].val.string_val) + else: +@@ -531,18 +613,18 @@ def main(): + print(response) + elif mode == 'set-update': + print('Performing SetRequest Update, encoding=JSON_IETF', ' to ', target, +- ' with the following gNMI Path\n', '-'*25, '\n', paths, json_value) +- response = _set(stub, paths, 'update', user, password, json_value) ++ ' with the following gNMI Path\n', '-'*25, '\n', paths, value_list) ++ response = _set(stub, prefix, paths, 'update', user, password, value_list) + print('The SetRequest response is below\n' + '-'*25 + '\n', response) + elif mode == 'set-replace': + print('Performing SetRequest Replace, encoding=JSON_IETF', ' to ', target, + ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _set(stub, paths, 'replace', user, password, json_value) ++ response = _set(stub, prefix, paths, 'replace', user, password, value_list) + print('The SetRequest response is below\n' + '-'*25 + '\n', response) + elif mode == 'set-delete': + print('Performing SetRequest Delete, encoding=JSON_IETF', ' to ', target, + ' with the following gNMI Path\n', '-'*25, '\n', paths) +- response = _set(stub, paths, 'delete', user, password, json_value) ++ response = _set(stub, prefix, paths, 'delete', user, password, value_list) + print('The SetRequest response is below\n' + '-'*25 + '\n', response) + elif mode == 'subscribe': + request_iterator = gen_request(paths, args, prefix) +@@ -552,6 +634,9 @@ def main(): + print("Client receives an exception '{}' indicating gNMI server is shut down and Exiting ..." + .format(err.details())) + sys.exit(GNMI_SERVER_UNAVAILABLE) ++ else: ++ print("GRPC error\n {}".format(err.details())) ++ sys.exit(1) + + + if __name__ == '__main__': +diff --git a/gnmi_cli_py/requirements.txt b/gnmi_cli_py/requirements.txt +index dab2db6..e32b3ff 100644 +--- a/gnmi_cli_py/requirements.txt ++++ b/gnmi_cli_py/requirements.txt +@@ -1,6 +1,6 @@ + enum34==1.1.6 + futures==3.2.0 +-grpcio==1.18.0 +-grpcio-tools==1.15.0 ++grpcio==1.41.1 ++grpcio-tools==1.41.1 + protobuf==3.6.1 --no-binary=protobuf + six==1.12.0 +-- +2.48.1.windows.1 + diff --git a/dockers/docker-ptf/gnxi-patches/series b/dockers/docker-ptf/gnxi-patches/series new file mode 100644 index 000000000000..d3c400ab929d --- /dev/null +++ b/dockers/docker-ptf/gnxi-patches/series @@ -0,0 +1,5 @@ +0001-add-xpath_target-option.patch +0002-Adding-support-for-subscribe-mode-1.patch +0003-gNMI_client-Add-an-option-to-trigger-memory-spike-on.patch +0004-Add-support-for-streaming-structured-events-in-night.patch +0005-Enhance-gnmi_cli_py-4.patch \ No newline at end of file