[db_migtrator] Add migration of FLEX_COUNTER_DELAY_STATUS during 1911->master upgrade + fast-reboot. Add UT. (sonic-net#2839)

Add migration of FLEX_COUNTER_DELAY_STATUS attribute of config_db FLEX_COUNTER_TABLE during the SONiC to SONiC upgrade + fast-reboot from older versions 201911 -> master.

This change is required for the fast-reboot procedure because without it the counters will be created during the init flow which will waste a lot of resources and cause data plane degradation of more than 30 seconds.

How I did it
Modify the db_migrator.py.

How to verify it
Add UT.


Signed-off-by: vadymhlushko-mlnx <[email protected]>

Author: vadymhlushko-mlnx

scripts/db_migrator.py

@@ -656,6 +656,20 @@ def update_edgezone_aggregator_config(self):
                 # Set new cable length values
                 self.configDB.set(self.configDB.CONFIG_DB, "CABLE_LENGTH|AZURE", intf, EDGEZONE_AGG_CABLE_LENGTH)
 
+    def migrate_config_db_flex_counter_delay_status(self):
+        """
+        Migrate "FLEX_COUNTER_TABLE|*": { "value": { "FLEX_COUNTER_DELAY_STATUS": "false" } }
+        Set FLEX_COUNTER_DELAY_STATUS true in case of fast-reboot
+        """
+
+        flex_counter_objects = self.configDB.get_keys('FLEX_COUNTER_TABLE')
+        for obj in flex_counter_objects:
+            flex_counter = self.configDB.get_entry('FLEX_COUNTER_TABLE', obj)
+            delay_status = flex_counter.get('FLEX_COUNTER_DELAY_STATUS')
+            if delay_status is None or delay_status == 'false':
+                flex_counter['FLEX_COUNTER_DELAY_STATUS'] = 'true'
+                self.configDB.mod_entry('FLEX_COUNTER_TABLE', obj, flex_counter)
+
     def version_unknown(self):
         """
         version_unknown tracks all SONiC versions that doesn't have a version
@@ -954,9 +968,21 @@ def version_4_0_1(self):
     def version_4_0_2(self):
         """
         Version 4_0_2.
-        This is the latest version for master branch
         """
         log.log_info('Handling version_4_0_2')
+
+        if self.stateDB.keys(self.stateDB.STATE_DB, "FAST_REBOOT|system"):
+            self.migrate_config_db_flex_counter_delay_status()
+
+        self.set_version('version_4_0_3')
+        return 'version_4_0_3'
+
+    def version_4_0_3(self):
+        """
+        Version 4_0_3.
+        This is the latest version for master branch
+        """
+        log.log_info('Handling version_4_0_3')
         return None
 
     def get_version(self):

tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json

@@ -0,0 +1,19 @@
+{
+    "VERSIONS|DATABASE": {
+        "VERSION": "version_4_0_3"
+    },
+    "FLEX_COUNTER_TABLE|ACL": {
+        "FLEX_COUNTER_STATUS": "true",
+        "FLEX_COUNTER_DELAY_STATUS": "true",
+        "POLL_INTERVAL": "10000"
+    },
+    "FLEX_COUNTER_TABLE|QUEUE": {
+        "FLEX_COUNTER_STATUS": "true",
+        "FLEX_COUNTER_DELAY_STATUS": "true",
+        "POLL_INTERVAL": "10000"
+    },
+    "FLEX_COUNTER_TABLE|PG_WATERMARK": {
+        "FLEX_COUNTER_STATUS": "false",
+        "FLEX_COUNTER_DELAY_STATUS": "true"
+    }
+}

tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json

@@ -0,0 +1,18 @@
+{
+    "VERSIONS|DATABASE": {
+        "VERSION": "version_1_0_1"
+    },
+    "FLEX_COUNTER_TABLE|ACL": {
+        "FLEX_COUNTER_STATUS": "true",
+        "FLEX_COUNTER_DELAY_STATUS": "true",
+        "POLL_INTERVAL": "10000"
+    },
+    "FLEX_COUNTER_TABLE|QUEUE": {
+        "FLEX_COUNTER_STATUS": "true",
+        "FLEX_COUNTER_DELAY_STATUS": "false",
+        "POLL_INTERVAL": "10000"
+    },
+    "FLEX_COUNTER_TABLE|PG_WATERMARK": {
+        "FLEX_COUNTER_STATUS": "false"
+    }
+}

tests/db_migrator_input/state_db/fast_reboot_upgrade.json

@@ -0,0 +1,5 @@
+{
+	"FAST_REBOOT|system": {
+		"enable": "true"
+	}
+}

tests/db_migrator_test.py

@@ -623,3 +623,38 @@ def test_warm_upgrade_t0_edgezone_aggregator_same_cable_length(self):
 
         diff = DeepDiff(resulting_table, expected_table, ignore_order=True)
         assert not diff
+
+
+class TestFastUpgrade_to_4_0_3(object):
+    @classmethod
+    def setup_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "2"
+        cls.config_db_tables_to_verify = ['FLEX_COUNTER_TABLE']
+        dbconnector.dedicated_dbs['STATE_DB'] = os.path.join(mock_db_path, 'state_db', 'fast_reboot_upgrade')
+
+    @classmethod
+    def teardown_class(cls):
+        os.environ['UTILITIES_UNIT_TESTING'] = "0"
+        dbconnector.dedicated_dbs['CONFIG_DB'] = None
+        dbconnector.dedicated_dbs['STATE_DB'] = None
+
+    def mock_dedicated_config_db(self, filename):
+        jsonfile = os.path.join(mock_db_path, 'config_db', filename)
+        dbconnector.dedicated_dbs['CONFIG_DB'] = jsonfile
+        db = Db()
+        return db
+
+    def check_config_db(self, result, expected):
+        for table in self.config_db_tables_to_verify:
+            assert result.get_table(table) == expected.get_table(table)
+
+    def test_fast_reboot_upgrade_to_4_0_3(self):
+        db_before_migrate = 'cross_branch_upgrade_to_4_0_3_input'
+        db_after_migrate = 'cross_branch_upgrade_to_4_0_3_expected'
+        device_info.get_sonic_version_info = get_sonic_version_info_mlnx
+        db = self.mock_dedicated_config_db(db_before_migrate)
+        import db_migrator
+        dbmgtr = db_migrator.DBMigrator(None)
+        dbmgtr.migrate()
+        expected_db = self.mock_dedicated_config_db(db_after_migrate)
+        assert not self.check_config_db(dbmgtr.configDB, expected_db.cfgdb)