support swss vstest in python3

Author: lguohan

tests/README.md

@@ -14,9 +14,9 @@ SWSS, Redis, and all the other required components run inside a virtual switch D
     ```
     sudo modprobe team
     sudo apt install net-tools ethtool vlan
-    sudo pip install docker zipp==2.2.1 pytest==4.6.9 flaky redis
+    sudo pip3 install docker zipp==2.2.1 pytest==4.6.9 flaky redis distro==1.4.0
     ```
-3. Install `python-swsscommon_1.0.0_amd64.deb`. You will need to install all the dependencies as well in the following order:
+3. Install `python3-swsscommon_1.0.0_amd64.deb`. You will need to install all the dependencies as well in the following order:
 
     ```
     sudo dpkg -i libnl-3-200_3.5.0-1_amd64.deb
@@ -30,7 +30,10 @@ SWSS, Redis, and all the other required components run inside a virtual switch D
 
     You can find the dependencies [here](https://sonic-jenkins.westus2.cloudapp.azure.com/job/vs/job/buildimage-vs-all/lastSuccessfulBuild/artifact/target/debs/stretch/), and get this package by:
     - [Building it from scratch](https://github.com/Azure/sonic-swss-common)
-    - [Downloading the latest build from Jenkins](https://sonic-jenkins.westus2.cloudapp.azure.com/job/common/job/sonic-swss-common-build/lastSuccessfulBuild/artifact/target/)
+    - Downloading the latest build from Jenkins
+      - [Debian](https://sonic-jenkins.westus2.cloudapp.azure.com/job/common/job/sonic-swss-common-build/lastSuccessfulBuild/artifact/target/)
+      - [Ubuntu 18.04](https://sonic-jenkins.westus2.cloudapp.azure.com/job/common/job/sonic-swss-common-build-ubuntu/lastSuccessfulBuild/artifact/target/)
+      - [Ubuntu 20.04](https://sonic-jenkins.westus2.cloudapp.azure.com/job/common/job/sonic-swss-common-build-ubuntu-20_04/lastSuccessfulBuild/artifact/target/)
 4. Load the `docker-sonic-vs.gz` file into docker. You can get the image by:
     - [Building it from scratch](https://github.com/Azure/sonic-buildimage)
     - [Downloading the latest build from Jenkins](https://sonic-jenkins.westus2.cloudapp.azure.com/job/vs/job/buildimage-vs-all/lastSuccessfulBuild/artifact/target/)

tests/conftest.py

@@ -6,10 +6,12 @@
 import redis
 import docker
 import pytest
-import commands
 import tarfile
-import StringIO
+import io
 import subprocess
+import sys
+if sys.version_info < (3, 0):
+    import commands
 
 from datetime import datetime
 from swsscommon import swsscommon
@@ -21,7 +23,10 @@
 from dvslib import dvs_policer
 
 def ensure_system(cmd):
-    (rc, output) = commands.getstatusoutput(cmd)
+    if sys.version_info < (3, 0):
+        (rc, output) = commands.getstatusoutput(cmd)
+    else:
+        (rc, output) = subprocess.getstatusoutput(cmd)
     if rc:
         raise RuntimeError('Failed to run command: %s. rc=%d. output: %s' % (cmd, rc, output))
 
@@ -138,17 +143,17 @@ def runcmd(self, cmd):
         try:
             out = subprocess.check_output("ip netns exec %s %s" % (self.nsname, cmd), stderr=subprocess.STDOUT, shell=True)
         except subprocess.CalledProcessError as e:
-            print "------rc={} for cmd: {}------".format(e.returncode, e.cmd)
-            print e.output.rstrip()
-            print "------"
+            print("------rc={} for cmd: {}------".format(e.returncode, e.cmd))
+            print(e.output.rstrip())
+            print("------")
             return e.returncode
         return 0
 
     def runcmd_async(self, cmd):
         return subprocess.Popen("ip netns exec %s %s" % (self.nsname, cmd), shell=True)
 
     def runcmd_output(self, cmd):
-        return subprocess.check_output("ip netns exec %s %s" % (self.nsname, cmd), shell=True)
+        return subprocess.check_output("ip netns exec %s %s" % (self.nsname, cmd), shell=True).decode('utf-8')
 
 class DockerVirtualSwitch(object):
     APP_DB_ID = 0
@@ -189,7 +194,10 @@ def __init__(self, name=None, imgname=None, keeptb=False, fakeplatform=None):
             for ctn in self.client.containers.list():
                 if ctn.name == name:
                     self.ctn = ctn
-                    (status, output) = commands.getstatusoutput("docker inspect --format '{{.HostConfig.NetworkMode}}' %s" % name)
+                    if sys.version_info < (3, 0):
+                        (status, output) = commands.getstatusoutput("docker inspect --format '{{.HostConfig.NetworkMode}}' %s" % name)
+                    else:
+                        (status, output) = subprocess.getstatusoutput("docker inspect --format '{{.HostConfig.NetworkMode}}' %s" % name)
                     ctn_sw_id = output.split(':')[1]
                     self.cleanup = False
             if self.ctn == None:
@@ -200,7 +208,10 @@ def __init__(self, name=None, imgname=None, keeptb=False, fakeplatform=None):
                 if ctn.id == ctn_sw_id or ctn.name == ctn_sw_id:
                     ctn_sw_name = ctn.name
 
-            (status, output) = commands.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % ctn_sw_name)
+            if sys.version_info < (3, 0):
+                (status, output) = commands.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % ctn_sw_name)
+            else:
+                (status, output) = subprocess.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % ctn_sw_name)
             self.ctn_sw_pid = int(output)
 
             # create virtual servers
@@ -216,7 +227,10 @@ def __init__(self, name=None, imgname=None, keeptb=False, fakeplatform=None):
         else:
             self.ctn_sw = self.client.containers.run('debian:jessie', privileged=True, detach=True,
                     command="bash", stdin_open=True)
-            (status, output) = commands.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % self.ctn_sw.name)
+            if sys.version_info < (3, 0):
+                (status, output) = commands.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % self.ctn_sw.name)
+            else:
+                (status, output) = subprocess.getstatusoutput("docker inspect --format '{{.State.Pid}}' %s" % self.ctn_sw.name)
             self.ctn_sw_pid = int(output)
 
             # create virtual server
@@ -286,7 +300,7 @@ def check_ready(self, timeout=30):
             # get process status
             res = self.ctn.exec_run("supervisorctl status")
             try:
-                out = res.output
+                out = res.output.decode('utf-8')
             except AttributeError:
                 out = res
             for l in out.split('\n'):
@@ -324,7 +338,7 @@ def net_cleanup(self):
 
         res = self.ctn.exec_run("ip link show")
         try:
-            out = res.output
+            out = res.output.decode('utf-8')
         except AttributeError:
             out = res
         for l in out.split('\n'):
@@ -337,7 +351,7 @@ def net_cleanup(self):
                 m = re.compile("(eth|lo|Bridge|Ethernet)").match(pname)
                 if not m:
                     self.ctn.exec_run("ip link del {}".format(pname))
-                    print "remove extra link {}".format(pname)
+                    print("remove extra link {}".format(pname))
         return
 
     def ctn_restart(self):
@@ -391,19 +405,19 @@ def runcmd(self, cmd):
         res = self.ctn.exec_run(cmd)
         try:
             exitcode = res.exit_code
-            out = res.output
+            out = res.output.decode('utf-8')
         except AttributeError:
             exitcode = 0
             out = res
         if exitcode != 0:
-            print "-----rc={} for cmd {}-----".format(exitcode, cmd)
-            print out.rstrip()
-            print "-----"
+            print("-----rc={} for cmd {}-----".format(exitcode, cmd))
+            print(out.rstrip())
+            print("-----")
 
         return (exitcode, out)
 
     def copy_file(self, path, filename):
-        tarstr = StringIO.StringIO()
+        tarstr = io.StringIO()
         tar = tarfile.open(fileobj=tarstr, mode="w")
         tar.add(filename, os.path.basename(filename))
         tar.close()
@@ -439,13 +453,15 @@ def add_log_marker(self, file=None):
         return marker
 
     def SubscribeAppDbObject(self, objpfx):
-        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB)
+        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB,
+                        encoding="utf-8", decode_responses=True)
         pubsub = r.pubsub()
         pubsub.psubscribe("__keyspace@0__:%s*" % objpfx)
         return pubsub
 
     def SubscribeAsicDbObject(self, objpfx):
-        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB)
+        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB,
+                        encoding="utf-8", decode_responses=True)
         pubsub = r.pubsub()
         pubsub.psubscribe("__keyspace@1__:ASIC_STATE:%s*" % objpfx)
         return pubsub
@@ -457,7 +473,7 @@ def CountSubscribedObjects(self, pubsub, ignore=None, timeout=10):
         while True and idle < timeout:
             message = pubsub.get_message()
             if message:
-                print message
+                print(message)
                 if ignore:
                     fds = message['channel'].split(':')
                     if fds[2] in ignore:
@@ -474,7 +490,8 @@ def CountSubscribedObjects(self, pubsub, ignore=None, timeout=10):
         return (nadd, ndel)
 
     def GetSubscribedAppDbObjects(self, pubsub, ignore=None, timeout=10):
-        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB)
+        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB,
+                        encoding="utf-8", decode_responses=True)
 
         addobjs = []
         delobjs = []
@@ -484,7 +501,7 @@ def GetSubscribedAppDbObjects(self, pubsub, ignore=None, timeout=10):
         while True and idle < timeout:
             message = pubsub.get_message()
             if message:
-                print message
+                print(message)
                 key = message['channel'].split(':', 1)[1]
                 # In producer/consumer_state_table scenarios, every entry will
                 # show up twice for every push/pop operation, so skip the second
@@ -517,7 +534,8 @@ def GetSubscribedAppDbObjects(self, pubsub, ignore=None, timeout=10):
 
 
     def GetSubscribedAsicDbObjects(self, pubsub, ignore=None, timeout=10):
-        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB)
+        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB,
+                        encoding="utf-8", decode_responses=True)
 
         addobjs = []
         delobjs = []
@@ -526,7 +544,7 @@ def GetSubscribedAsicDbObjects(self, pubsub, ignore=None, timeout=10):
         while True and idle < timeout:
             message = pubsub.get_message()
             if message:
-                print message
+                print(message)
                 key = message['channel'].split(':', 1)[1]
                 if ignore:
                     fds = message['channel'].split(':')
@@ -548,7 +566,8 @@ def GetSubscribedAsicDbObjects(self, pubsub, ignore=None, timeout=10):
 
     def SubscribeDbObjects(self, dbobjs):
         # assuming all the db object pairs are in the same db instance
-        r = redis.Redis(unix_socket_path=self.redis_sock)
+        r = redis.Redis(unix_socket_path=self.redis_sock, encoding="utf-8",
+                        decode_responses=True)
         pubsub = r.pubsub()
         substr = ""
         for db, obj in dbobjs:
@@ -860,7 +879,8 @@ def setReadOnlyAttr(self, obj, attr, val):
         assert len(keys) == 1
 
         swVid = keys[0]
-        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB)
+        r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.ASIC_DB,
+                        encoding="utf-8", decode_responses=True)
         swRid = r.hget("VIDTORID", swVid)
 
         assert swRid is not None
@@ -871,7 +891,8 @@ def setReadOnlyAttr(self, obj, attr, val):
         fvp = swsscommon.FieldValuePairs([(attr, val)])
         key = "SAI_OBJECT_TYPE_SWITCH:" + swRid
 
-        ntf.send("set_ro", key, fvp)
+        # explicit convert unicode string to str for python2
+        ntf.send("set_ro", str(key), fvp)
 
     def create_acl_table(self, table, type, ports):
         tbl = swsscommon.Table(self.cdb, "ACL_TABLE")

tests/dvslib/dvs_vlan.py

@@ -1,4 +1,4 @@
-from dvs_database import DVSDatabase
+from .dvs_database import DVSDatabase
 
 class DVSVlan(object):
     def __init__(self, adb, cdb, sdb, cntrdb, appdb):

tests/port_dpb.py

@@ -1,5 +1,4 @@
 from swsscommon import swsscommon
-import redis
 import time
 import os
 import pytest
@@ -26,7 +25,7 @@ def __init__(self, dvs, name = None):
         self._app_db_ptbl = swsscommon.Table(self._app_db, swsscommon.APP_PORT_TABLE_NAME)
         self._asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0)
         self._asic_db_ptbl = swsscommon.Table(self._asic_db, "ASIC_STATE:SAI_OBJECT_TYPE_PORT")
-        self._counters_db = redis.Redis(unix_socket_path=self._dvs.redis_sock, db=swsscommon.COUNTERS_DB)
+        self._counters_db = dvs.get_counters_db()
         self._dvs_asic_db = dvs.get_asic_db()
 
     def set_name(self, name):
@@ -87,7 +86,7 @@ def get_oid(self):
         return self._oid
 
     def print_port(self):
-        print "Port: %s Lanes: %s Speed: %d, Index: %d"%(self._name, self._lanes, self._speed, self._index)
+        print("Port: %s Lanes: %s Speed: %d, Index: %d"%(self._name, self._lanes, self._speed, self._index))
 
     def port_merge(self, child_ports):
         child_ports.sort(key=lambda x: x.get_port_num())
@@ -111,7 +110,7 @@ def port_split(self, child_ports):
         child_port_list = []
         port_num = self.get_port_num()
         num_lanes = len(self._lanes)
-        offset = num_lanes/child_ports;
+        offset = num_lanes//child_ports
         lanes_per_child = offset
         for i in range(child_ports):
             child_port_num = port_num + (i * offset)
@@ -120,7 +119,7 @@ def port_split(self, child_ports):
             child_port_lanes = []
             for j in range(lanes_per_child):
                 child_port_lanes.append(self._lanes[(i*offset)+j])
-            child_port_speed = self._speed/child_ports
+            child_port_speed = self._speed//child_ports
             child_port_index = self._index
 
             child_port = Port(self._dvs, child_port_name)
@@ -172,7 +171,11 @@ def exists_in_app_db(self):
         return status
 
     def sync_oid(self):
-        self._oid = self._counters_db.hget("COUNTERS_PORT_NAME_MAP", self.get_name())
+        fvs = dict(self._counters_db.get_entry("COUNTERS_PORT_NAME_MAP", ""))
+        try:
+            self._oid = fvs[self.get_name()]
+        except KeyError:
+            self._oid = None
 
     """
         Expectation of the caller is that the port does exist in ASIC DB.
@@ -218,7 +221,7 @@ def verify_asic_db(self):
         (status, fvs) = self._asic_db_ptbl.get(self.get_oid())
         assert(status == True)
         fvs_dict = self.get_fvs_dict(fvs)
-        if (fvs_dict.has_key("SAI_PORT_ATTR_HW_LANE_LIST")):
+        if "SAI_PORT_ATTR_HW_LANE_LIST" in fvs_dict:
             assert(fvs_dict['SAI_PORT_ATTR_HW_LANE_LIST'] == self.get_lanes_asic_db_str())
         assert(fvs_dict['SAI_PORT_ATTR_SPEED'] == str(self.get_speed()))