Skip to content

Commit 9677447

Browse files
ganglyuganglyu
authored
Run yang validation in db migrator (#3102)
What I did Add unit test/runtime alert for db_migrator.py/load_minigraph: ConfigDB passing yang validation How I did it Use subprocess to run yang validation, and then warm-reboot performance will not change. Check /etc/sonic/mgmt_test_mark, this mark_file means this device is under end to end test. For unit test, db_migrator will wait for validation result and raise exception. For end to end test, db_migrator will wait for validation result and raise exception. How to verify it Run unit test and run end to end test
1 parent 9708f52 commit 9677447

13 files changed

+142
-20
lines changed

‎scripts/config_validator.py

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env python3
2+
import json
3+
import argparse
4+
import sonic_yang
5+
6+
from sonic_py_common import logger
7+
8+
YANG_MODELS_DIR = "/usr/local/yang-models"
9+
SYSLOG_IDENTIFIER = 'config_validator'
10+
11+
# Global logger instance
12+
log = logger.Logger(SYSLOG_IDENTIFIER)
13+
14+
15+
def main():
16+
parser = argparse.ArgumentParser()
17+
parser.add_argument('-c',
18+
dest='config',
19+
metavar='config file',
20+
type=str,
21+
required=True,
22+
help='the config file to be validated',
23+
default=None)
24+
25+
args = parser.parse_args()
26+
config_file = args.config
27+
with open(config_file) as fp:
28+
config = json.load(fp)
29+
# Run yang validation
30+
yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR)
31+
yang_parser.loadYangModel()
32+
try:
33+
yang_parser.loadData(configdbJson=config)
34+
yang_parser.validate_data_tree()
35+
except sonic_yang.SonicYangException as e:
36+
log.log_error("Yang validation failed: " + str(e))
37+
raise
38+
if len(yang_parser.tablesWithOutYang):
39+
log.log_error("Tables without yang models: " + str(yang_parser.tablesWithOutYang))
40+
raise Exception("Tables without yang models: " + str(yang_parser.tablesWithOutYang))
41+
42+
43+
if __name__ == "__main__":
44+
main()

‎scripts/db_migrator.py

+29
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import sys
77
import traceback
88
import re
9+
import subprocess
910

1011
from sonic_py_common import device_info, logger
1112
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
@@ -1304,6 +1305,34 @@ def migrate(self):
13041305
version = next_version
13051306
# Perform common migration ops
13061307
self.common_migration_ops()
1308+
# Perform yang validation
1309+
self.validate()
1310+
1311+
def validate(self):
1312+
config = self.configDB.get_config()
1313+
# Fix table key in tuple
1314+
for table_name, table in config.items():
1315+
new_table = {}
1316+
hit = False
1317+
for table_key, table_val in table.items():
1318+
if isinstance(table_key, tuple):
1319+
new_key = "|".join(table_key)
1320+
new_table[new_key] = table_val
1321+
hit = True
1322+
else:
1323+
new_table[table_key] = table_val
1324+
if hit:
1325+
config[table_name] = new_table
1326+
config_file = "/tmp/validate.json"
1327+
with open(config_file, 'w') as fp:
1328+
json.dump(config, fp)
1329+
process = subprocess.Popen(["config_validator.py", "-c", config_file])
1330+
# Check validation result for unit test
1331+
# Check validation result for end to end test
1332+
mark_file = "/etc/sonic/mgmt_test_mark"
1333+
if os.environ.get("UTILITIES_UNIT_TESTING", "0") == "2" or os.path.exists(mark_file):
1334+
ret = process.wait()
1335+
assert ret == 0, "Yang validation failed"
13071336

13081337
def main():
13091338
try:

‎setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@
116116
'scripts/buffershow',
117117
'scripts/coredump-compress',
118118
'scripts/configlet',
119+
'scripts/config_validator.py',
119120
'scripts/db_migrator.py',
120121
'scripts/decode-syseeprom',
121122
'scripts/dropcheck',

‎tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@
33
"VERSION": "version_4_0_3"
44
},
55
"FLEX_COUNTER_TABLE|ACL": {
6-
"FLEX_COUNTER_STATUS": "true",
6+
"FLEX_COUNTER_STATUS": "enable",
77
"FLEX_COUNTER_DELAY_STATUS": "true",
88
"POLL_INTERVAL": "10000"
99
},
1010
"FLEX_COUNTER_TABLE|QUEUE": {
11-
"FLEX_COUNTER_STATUS": "true",
11+
"FLEX_COUNTER_STATUS": "enable",
1212
"FLEX_COUNTER_DELAY_STATUS": "true",
1313
"POLL_INTERVAL": "10000"
1414
},
1515
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
16-
"FLEX_COUNTER_STATUS": "false",
16+
"FLEX_COUNTER_STATUS": "disable",
1717
"FLEX_COUNTER_DELAY_STATUS": "true"
1818
}
1919
}

‎tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33
"VERSION": "version_1_0_1"
44
},
55
"FLEX_COUNTER_TABLE|ACL": {
6-
"FLEX_COUNTER_STATUS": "true",
6+
"FLEX_COUNTER_STATUS": "enable",
77
"FLEX_COUNTER_DELAY_STATUS": "true",
88
"POLL_INTERVAL": "10000"
99
},
1010
"FLEX_COUNTER_TABLE|QUEUE": {
11-
"FLEX_COUNTER_STATUS": "true",
11+
"FLEX_COUNTER_STATUS": "enable",
1212
"FLEX_COUNTER_DELAY_STATUS": "false",
1313
"POLL_INTERVAL": "10000"
1414
},
1515
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
16-
"FLEX_COUNTER_STATUS": "false"
16+
"FLEX_COUNTER_STATUS": "disable"
1717
}
1818
}

‎tests/db_migrator_input/config_db/portchannel-expected.json

-4
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,24 @@
11
{
22
"PORTCHANNEL|PortChannel0": {
33
"admin_status": "up",
4-
"members@": "Ethernet0,Ethernet4",
54
"min_links": "2",
65
"mtu": "9100",
76
"lacp_key": "auto"
87
},
98
"PORTCHANNEL|PortChannel1": {
109
"admin_status": "up",
11-
"members@": "Ethernet8,Ethernet12",
1210
"min_links": "2",
1311
"mtu": "9100",
1412
"lacp_key": "auto"
1513
},
1614
"PORTCHANNEL|PortChannel0123": {
1715
"admin_status": "up",
18-
"members@": "Ethernet16",
1916
"min_links": "1",
2017
"mtu": "9100",
2118
"lacp_key": "auto"
2219
},
2320
"PORTCHANNEL|PortChannel0011": {
2421
"admin_status": "up",
25-
"members@": "Ethernet20,Ethernet24",
2622
"min_links": "2",
2723
"mtu": "9100",
2824
"lacp_key": "auto"

‎tests/db_migrator_input/config_db/portchannel-input.json

-4
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,21 @@
11
{
22
"PORTCHANNEL|PortChannel0": {
33
"admin_status": "up",
4-
"members@": "Ethernet0,Ethernet4",
54
"min_links": "2",
65
"mtu": "9100"
76
},
87
"PORTCHANNEL|PortChannel1": {
98
"admin_status": "up",
10-
"members@": "Ethernet8,Ethernet12",
119
"min_links": "2",
1210
"mtu": "9100"
1311
},
1412
"PORTCHANNEL|PortChannel0123": {
1513
"admin_status": "up",
16-
"members@": "Ethernet16",
1714
"min_links": "1",
1815
"mtu": "9100"
1916
},
2017
"PORTCHANNEL|PortChannel0011": {
2118
"admin_status": "up",
22-
"members@": "Ethernet20,Ethernet24",
2319
"min_links": "2",
2420
"mtu": "9100"
2521
},

‎tests/db_migrator_input/config_db/qos_map_table_expected.json

+9-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@
2929
"pfc_to_queue_map": "AZURE",
3030
"tc_to_pg_map": "AZURE",
3131
"tc_to_queue_map": "AZURE"
32-
}
32+
},
33+
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
34+
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
35+
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
36+
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
37+
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
38+
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
39+
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
40+
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
3341
}
3442

‎tests/db_migrator_input/config_db/qos_map_table_input.json

+9-1
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,13 @@
2727
"pfc_to_queue_map": "AZURE",
2828
"tc_to_pg_map": "AZURE",
2929
"tc_to_queue_map": "AZURE"
30-
}
30+
},
31+
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
32+
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
33+
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
34+
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
35+
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
36+
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
37+
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
38+
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
3139
}

‎tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json

+11-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"profile": "NULL"
1313
},
1414
"BUFFER_PG|Ethernet8|3-4": {
15-
"profile": "customized_lossless_profile"
15+
"profile": "customized_ingress_lossless_profile"
1616
},
1717
"BUFFER_PG|Ethernet12|0": {
1818
"profile": "ingress_lossy_profile"
@@ -103,6 +103,11 @@
103103
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
104104
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
105105
},
106+
"BUFFER_PROFILE|customized_egress_lossless_profile": {
107+
"dynamic_th": "7",
108+
"pool": "egress_lossless_pool",
109+
"size": "0"
110+
},
106111
"BUFFER_PROFILE|egress_lossless_profile": {
107112
"dynamic_th": "7",
108113
"pool": "egress_lossless_pool",
@@ -118,6 +123,11 @@
118123
"pool": "ingress_lossless_pool",
119124
"size": "0"
120125
},
126+
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
127+
"dynamic_th": "7",
128+
"pool": "ingress_lossless_pool",
129+
"size": "0"
130+
},
121131
"BUFFER_PROFILE|ingress_lossy_profile": {
122132
"dynamic_th": "3",
123133
"pool": "ingress_lossy_pool",

‎tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json

+11-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"profile": "NULL"
44
},
55
"BUFFER_PG|Ethernet8|3-4": {
6-
"profile": "customized_lossless_profile"
6+
"profile": "customized_ingress_lossless_profile"
77
},
88
"BUFFER_PG|Ethernet12|0": {
99
"profile": "ingress_lossy_profile"
@@ -55,6 +55,11 @@
5555
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
5656
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
5757
},
58+
"BUFFER_PROFILE|customized_egress_lossless_profile": {
59+
"dynamic_th": "7",
60+
"pool": "egress_lossless_pool",
61+
"size": "0"
62+
},
5863
"BUFFER_PROFILE|egress_lossless_profile": {
5964
"dynamic_th": "7",
6065
"pool": "egress_lossless_pool",
@@ -65,6 +70,11 @@
6570
"pool": "egress_lossy_pool",
6671
"size": "9216"
6772
},
73+
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
74+
"dynamic_th": "7",
75+
"pool": "ingress_lossless_pool",
76+
"size": "0"
77+
},
6878
"BUFFER_PROFILE|ingress_lossless_profile": {
6979
"dynamic_th": "7",
7080
"pool": "ingress_lossless_pool",

‎tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json

+11-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"profile": "NULL"
1313
},
1414
"BUFFER_PG|Ethernet8|3-4": {
15-
"profile": "customized_lossless_profile"
15+
"profile": "customized_ingress_lossless_profile"
1616
},
1717
"BUFFER_PG|Ethernet12|0": {
1818
"profile": "ingress_lossy_profile"
@@ -99,6 +99,11 @@
9999
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
100100
"profile_list": "ingress_lossless_profile"
101101
},
102+
"BUFFER_PROFILE|customized_egress_lossless_profile": {
103+
"dynamic_th": "7",
104+
"pool": "egress_lossless_pool",
105+
"size": "0"
106+
},
102107
"BUFFER_PROFILE|egress_lossless_profile": {
103108
"dynamic_th": "7",
104109
"pool": "egress_lossless_pool",
@@ -109,6 +114,11 @@
109114
"pool": "egress_lossy_pool",
110115
"size": "9216"
111116
},
117+
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
118+
"dynamic_th": "7",
119+
"pool": "ingress_lossless_pool",
120+
"size": "0"
121+
},
112122
"BUFFER_PROFILE|ingress_lossless_profile": {
113123
"dynamic_th": "7",
114124
"pool": "ingress_lossless_pool",

‎tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json

+11-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"profile": "NULL"
44
},
55
"BUFFER_PG|Ethernet8|3-4": {
6-
"profile": "customized_lossless_profile"
6+
"profile": "customized_ingress_lossless_profile"
77
},
88
"BUFFER_PG|Ethernet12|0": {
99
"profile": "ingress_lossy_profile"
@@ -51,6 +51,11 @@
5151
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
5252
"profile_list": "ingress_lossless_profile"
5353
},
54+
"BUFFER_PROFILE|customized_egress_lossless_profile": {
55+
"dynamic_th": "7",
56+
"pool": "egress_lossless_pool",
57+
"size": "0"
58+
},
5459
"BUFFER_PROFILE|egress_lossless_profile": {
5560
"dynamic_th": "7",
5661
"pool": "egress_lossless_pool",
@@ -61,6 +66,11 @@
6166
"pool": "egress_lossy_pool",
6267
"size": "9216"
6368
},
69+
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
70+
"dynamic_th": "7",
71+
"pool": "ingress_lossless_pool",
72+
"size": "0"
73+
},
6474
"BUFFER_PROFILE|ingress_lossless_profile": {
6575
"dynamic_th": "7",
6676
"pool": "ingress_lossless_pool",

0 commit comments

Comments
 (0)