neighbor syncd warm restart test cases (sonic-net#2)

* Add neighbor table warmrestart test cases

* Change the interface to be used for neighbor table to avoid conflict

* Add syslog and sairedis checks for neighbor sync

Author: zhenggen-xu

tests/test_warm_reboot.py

@@ -396,3 +396,359 @@ def test_VlanMgrdPortSyncdWarmRestart(dvs):
     # To clean up test env.
     dvs.runcmd("redis-cli flushall")
 
+
+# function to check the restart counter
+def check_restart_cnt(warmtbl, restart_cnt):
+    keys = warmtbl.getKeys()
+    print(keys)
+    for key in keys:
+        (status, fvs) = warmtbl.get(key)
+        assert status == True
+        for fv in fvs:
+            if fv[0] == "restart_count":
+                assert fv[1] == str(restart_cnt)
+            elif fv[0] == "state_restored":
+                assert fv[1] == "true"
+
+
+# function to stop swss service and clear syslog and sairedis records
+def stop_swss_clear_syslog_sairedis(dvs, save_number):
+    dvs.runcmd("/usr/bin/stop_swss.sh")
+    time.sleep(3)
+    dvs.runcmd("mv /var/log/swss/sairedis.rec /var/log/swss/sairedis.rec.back1")
+    dvs.runcmd("cp /var/log/syslog /var/log/syslog.back{}".format(save_number))
+    dvs.runcmd(['sh', '-c', '> /var/log/syslog'])
+
+
+# function to check neighbor entry reconciliation status written in syslog
+def check_syslog_for_neighbor_entry(dvs, new_cnt, delete_cnt, iptype):
+    # check reconciliation results (new or delete entries) for ipv4 and ipv6
+    if iptype == "ipv4":
+        num = dvs.runcmd(['sh', '-c', 'grep neighsyncd /var/log/syslog| grep cache-state:NEW | grep IPv4 | wc -l'])
+        assert num.strip() == str(new_cnt)
+        num = dvs.runcmd(['sh', '-c', 'grep neighsyncd /var/log/syslog| grep cache-state:DELETE | grep IPv4 | wc -l'])
+        assert num.strip() == str(delete_cnt)
+    elif iptype == "ipv6":
+        num = dvs.runcmd(['sh', '-c', 'grep neighsyncd /var/log/syslog| grep cache-state:NEW | grep IPv6 | wc -l'])
+        assert num.strip() == str(new_cnt)
+        num = dvs.runcmd(['sh', '-c', 'grep neighsyncd /var/log/syslog| grep cache-state:DELETE | grep IPv6 | wc -l'])
+        assert num.strip() == str(delete_cnt)
+    else:
+        assert "iptype is unknown" == ""
+
+
+# function to check sairedis record for neighbor entries
+def check_sairedis_for_neighbor_entry(dvs, create_cnt, set_cnt, remove_cnt):
+    # check create/set/remove operations for neighbor entries during warm restart
+    num = dvs.runcmd(['sh', '-c', 'grep \|c\| /var/log/swss/sairedis.rec | grep NEIGHBOR_ENTRY | wc -l'])
+    assert num.strip() == str(create_cnt)
+    num = dvs.runcmd(['sh', '-c', 'grep \|s\| /var/log/swss/sairedis.rec | grep NEIGHBOR_ENTRY | wc -l'])
+    assert num.strip() == str(set_cnt)
+    num = dvs.runcmd(['sh', '-c', 'grep \|r\| /var/log/swss/sairedis.rec | grep NEIGHBOR_ENTRY | wc -l'])
+    assert num.strip() == str(remove_cnt)
+
+
+def test_swss_neighbor_syncup(dvs):
+    # syncd warm start with temp view not supported yet
+    if dvs.tmpview == True:
+        return
+
+    # previous warm restart cnt
+    restart_cnt = 4
+
+    # Prepare neighbor entry before swss stop
+    appl_db = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0)
+    asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0)
+    conf_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0)
+
+    #
+    # Testcase1:
+    # Add neighbor entries in linux kernel, appDB should get all of them
+    #
+
+    # create neighbor entries (4 ipv4 and 4 ip6, two each on each interface) in linux kernel
+    intfs = ["Ethernet24", "Ethernet28"]
+    #enable ipv6 on docker
+    dvs.runcmd("sysctl net.ipv6.conf.all.disable_ipv6=0")
+
+    dvs.runcmd("ifconfig {} 24.0.0.1/24 up".format(intfs[0]))
+    dvs.runcmd("ip -6 addr add 2400::1/64 dev {}".format(intfs[0]))
+
+    dvs.runcmd("ifconfig {} 28.0.0.1/24 up".format(intfs[1]))
+    dvs.runcmd("ip -6 addr add 2800::1/64 dev {}".format(intfs[1]))
+
+    ips = ["24.0.0.2", "24.0.0.3", "28.0.0.2", "28.0.0.3"]
+    v6ips = ["2400::2", "2400::3", "2800::2", "2800::3"]
+
+    macs = ["00:00:00:00:24:02", "00:00:00:00:24:03", "00:00:00:00:28:02", "00:00:00:00:28:03"]
+
+    for i in range(len(ips)):
+        dvs.runcmd("ip neigh add {} dev {} lladdr {}".format(ips[i], intfs[i%2], macs[i]))
+
+    for i in range(len(v6ips)):
+        dvs.runcmd("ip -6 neigh add {} dev {} lladdr {}".format(v6ips[i], intfs[i%2], macs[i]))
+
+    time.sleep(1)
+
+    # Check the neighbor entries are inserted correctly
+    db = swsscommon.DBConnector(0, dvs.redis_sock, 0)
+    tbl = swsscommon.Table(db, "NEIGH_TABLE")
+
+    for i in range(len(ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], ips[i]))
+        assert status == True
+
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv4"
+
+    for i in range(len(v6ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], v6ips[i]))
+        assert status == True
+
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv6"
+
+    #
+    # Testcase 2:
+    # Restart swss without change neighbor entries, nothing should be sent to appDB or sairedis,
+    # appDB should be kept the same.
+    #
+
+    # stop swss service and clear syslog and sairedis.rec
+    stop_swss_clear_syslog_sairedis(dvs, 1)
+
+    dvs.runcmd("/usr/bin/start_swss.sh")
+    time.sleep(10)
+
+    # check restart_count for each process in SWSS
+    restart_cnt += 1
+    warmtbl = swsscommon.Table(appl_db, "WARM_START_TABLE")
+    check_restart_cnt(warmtbl, restart_cnt)
+
+    # Check the neighbor entries are still in appDB correctly
+    for i in range(len(ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], ips[i]))
+        assert status == True
+
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv4"
+
+    for i in range(len(v6ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], v6ips[i]))
+        assert status == True
+
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv6"
+
+    # check syslog and sairedis.rec file for activities
+    check_syslog_for_neighbor_entry(dvs, 0, 0, "ipv4")
+    check_syslog_for_neighbor_entry(dvs, 0, 0, "ipv6")
+    check_sairedis_for_neighbor_entry(dvs, 0, 0, 0)
+
+    #
+    # Testcase 3:
+    # stop swss, delete even nummber ipv4/ipv6 neighbor entries from each interface, warm start swss.
+    # the neighsyncd is supposed to sync up the entries from kernel after warm restart
+    # note: there was an issue for neighbor delete, it will be marked as FAILED instead of deleted in kernel
+    #       but it will send netlink message to be removed from appDB, so it works ok here,
+    #       just that if we want to add the same neighbor again, use "change" instead of "add"
+
+    # stop swss service and clear syslog and sairedis.rec
+    stop_swss_clear_syslog_sairedis(dvs, 2)
+
+    # deledelete even nummber of ipv4/ipv6 neighbor entries from each interface
+    for i in range(0, len(ips), 2):
+        dvs.runcmd("ip neigh del {} dev {}".format(ips[i], intfs[i%2]))
+
+    for i in range(0, len(v6ips), 2):
+        dvs.runcmd("ip -6 neigh del {} dev {}".format(v6ips[i], intfs[i%2]))
+
+    # start swss service again
+    dvs.runcmd("/usr/bin/start_swss.sh")
+    time.sleep(10)
+
+    # check restart_count for each process in SWSS
+    restart_cnt += 1
+    warmtbl = swsscommon.Table(appl_db, "WARM_START_TABLE")
+    check_restart_cnt(warmtbl, restart_cnt)
+
+    # check ipv4 and ipv6 neighbors
+    for i in range(len(ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], ips[i]))
+        #should not see deleted neighbor entries
+        if i %2 == 0:
+            assert status == False
+            continue
+        else:
+            assert status == True
+
+        #undeleted entries should still be there.
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv4"
+
+    for i in range(len(v6ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], v6ips[i]))
+        #should not see deleted neighbor entries
+        if i %2 == 0:
+            assert status == False
+            continue
+        else:
+            assert status == True
+
+        #undeleted entries should still be there.
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv6"
+
+    # check syslog and sairedis.rec file for activities
+    # 2 deletes each for ipv4 and ipv6
+    # 4 remove actions in sairedis
+    check_syslog_for_neighbor_entry(dvs, 0, 2, "ipv4")
+    check_syslog_for_neighbor_entry(dvs, 0, 2, "ipv6")
+    check_sairedis_for_neighbor_entry(dvs, 0, 0, 4)
+
+    #
+    # Testcase 4:
+    # Stop swss, add even nummber of ipv4/ipv6 neighbor entries to each interface again,
+    # use "change" due to the kernel behaviour, start swss.
+    # The neighsyncd is supposed to sync up the entries from kernel after warm restart
+
+    # stop swss service and clear syslog and sairedis.rec
+    stop_swss_clear_syslog_sairedis(dvs, 3)
+
+    # add even nummber of ipv4/ipv6 neighbor entries to each interface
+    for i in range(0, len(ips), 2):
+        dvs.runcmd("ip neigh change {} dev {} lladdr {}".format(ips[i], intfs[i%2], macs[i]))
+
+    for i in range(0, len(v6ips), 2):
+        dvs.runcmd("ip -6 neigh change {} dev {} lladdr {}".format(v6ips[i], intfs[i%2], macs[i]))
+
+    # start swss service again
+    dvs.runcmd("/usr/bin/start_swss.sh")
+    time.sleep(10)
+
+    # check restart_count for each process in SWSS
+    restart_cnt += 1
+    check_restart_cnt(warmtbl, restart_cnt)
+
+    # check ipv4 and ipv6 neighbors, should see all neighbors
+    for i in range(len(ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], ips[i]))
+        assert status == True
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv4"
+
+    for i in range(len(v6ips)):
+        (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], v6ips[i]))
+        assert status == True
+        for v in fvs:
+            if v[0] == "neigh":
+                assert v[1] == macs[i]
+            if v[0] == "family":
+                assert v[1] == "IPv6"
+
+    # check syslog and sairedis.rec file for activities
+    # 2 news entries for ipv4 and ipv6 each
+    # 4 create actions for sairedis
+    check_syslog_for_neighbor_entry(dvs, 2, 0, "ipv4")
+    check_syslog_for_neighbor_entry(dvs, 2, 0, "ipv6")
+    check_sairedis_for_neighbor_entry(dvs, 4, 0, 0)
+
+    #
+    # Testcase 5:
+    # Even number of ip4/6 neigbors updated with new mac.
+    # Odd number of ipv4/6 neighbors removed and added to different interfaces.
+    # neighbor syncd should sync it up after warm restart
+
+    # stop swss service and clear syslog and sairedis.rec
+    stop_swss_clear_syslog_sairedis(dvs, 4)
+
+    # Even number of ip4/6 neigbors updated with new mac.
+    # Odd number of ipv4/6 neighbors removed and added to different interfaces.
+    newmacs = ["00:00:00:01:12:02", "00:00:00:01:12:03", "00:00:00:01:16:02", "00:00:00:01:16:03"]
+
+    for i in range(len(ips)):
+        if i % 2 == 0:
+            dvs.runcmd("ip neigh change {} dev {} lladdr {}".format(ips[i], intfs[i%2], newmacs[i]))
+        else:
+            dvs.runcmd("ip neigh del {} dev {}".format(ips[i], intfs[i%2]))
+            dvs.runcmd("ip neigh add {} dev {} lladdr {}".format(ips[i], intfs[1-i%2], macs[i]))
+
+    for i in range(len(v6ips)):
+        if i % 2 == 0:
+            dvs.runcmd("ip -6 neigh change {} dev {} lladdr {}".format(v6ips[i], intfs[i%2], newmacs[i]))
+        else:
+            dvs.runcmd("ip -6 neigh del {} dev {}".format(v6ips[i], intfs[i%2]))
+            dvs.runcmd("ip -6 neigh add {} dev {} lladdr {}".format(v6ips[i], intfs[1-i%2], macs[i]))
+
+    # start swss service again
+    dvs.runcmd("/usr/bin/start_swss.sh")
+    time.sleep(10)
+
+    # check restart_count for each process in SWSS
+    restart_cnt += 1
+    check_restart_cnt(warmtbl, restart_cnt)
+
+    # check ipv4 and ipv6 neighbors, should see all neighbors with updated info
+    for i in range(len(ips)):
+        if i % 2 == 0:
+            (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], ips[i]))
+            assert status == True
+            for v in fvs:
+                if v[0] == "neigh":
+                    assert v[1] == newmacs[i]
+                if v[0] == "family":
+                    assert v[1] == "IPv4"
+        else:
+            (status, fvs) = tbl.get("{}:{}".format(intfs[1-i%2], ips[i]))
+            assert status == True
+            for v in fvs:
+                if v[0] == "neigh":
+                    assert v[1] == macs[i]
+                if v[0] == "family":
+                    assert v[1] == "IPv4"
+
+    for i in range(len(v6ips)):
+        if i % 2 == 0:
+            (status, fvs) = tbl.get("{}:{}".format(intfs[i%2], v6ips[i]))
+            assert status == True
+            for v in fvs:
+                if v[0] == "neigh":
+                    assert v[1] == newmacs[i]
+                if v[0] == "family":
+                    assert v[1] == "IPv6"
+        else:
+            (status, fvs) = tbl.get("{}:{}".format(intfs[1-i%2], v6ips[i]))
+            assert status == True
+            for v in fvs:
+                if v[0] == "neigh":
+                    assert v[1] == macs[i]
+                if v[0] == "family":
+                    assert v[1] == "IPv6"
+
+    # check syslog and sairedis.rec file for activities
+    # 4 news, 2 deletes for ipv4 and ipv6 each
+    # 8 create, 4 set, 4 removes for sairedis
+    check_syslog_for_neighbor_entry(dvs, 4, 2, "ipv4")
+    check_syslog_for_neighbor_entry(dvs, 4, 2, "ipv6")
+    check_sairedis_for_neighbor_entry(dvs, 4, 4, 4)