Skip to content

Commit be4689d

Browse files
JunhongMaoJunhongMao
authored
Update for the procedures for insertion/hot swap of Switch Fabric Module(SFM) by using "config chassis modules shutdown/startup" commands (#3283)
sudo config chassis modules shutdown/startup <module name> The HLD for Shutdown and Startup of the Fabric Module is below: sonic-net/SONiC#1694
1 parent 8217441 commit be4689d

File tree

2 files changed

+213
-1
lines changed

2 files changed

+213
-1
lines changed

‎config/chassis_modules.py

100644100755
+100-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
#!/usr/sbin/env python
22

33
import click
4-
4+
import time
5+
import re
6+
import subprocess
57
import utilities_common.cli as clicommon
68

9+
TIMEOUT_SECS = 10
10+
11+
712
#
813
# 'chassis_modules' group ('config chassis_modules ...')
914
#
@@ -17,6 +22,81 @@ def modules():
1722
"""Configure chassis modules"""
1823
pass
1924

25+
26+
def get_config_module_state(db, chassis_module_name):
27+
config_db = db.cfgdb
28+
fvs = config_db.get_entry('CHASSIS_MODULE', chassis_module_name)
29+
if not fvs:
30+
return 'up'
31+
else:
32+
return fvs['admin_status']
33+
34+
35+
#
36+
# Name: check_config_module_state_with_timeout
37+
# return: True: timeout, False: not timeout
38+
#
39+
def check_config_module_state_with_timeout(ctx, db, chassis_module_name, state):
40+
counter = 0
41+
while get_config_module_state(db, chassis_module_name) != state:
42+
time.sleep(1)
43+
counter += 1
44+
if counter >= TIMEOUT_SECS:
45+
ctx.fail("get_config_module_state {} timeout".format(chassis_module_name))
46+
return True
47+
return False
48+
49+
50+
def get_asic_list_from_db(chassisdb, chassis_module_name):
51+
asic_list = []
52+
asics_keys_list = chassisdb.keys("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE*")
53+
for asic_key in asics_keys_list:
54+
name = chassisdb.get("CHASSIS_STATE_DB", asic_key, "name")
55+
if name == chassis_module_name:
56+
asic_id = int(re.search(r"(\d+)$", asic_key).group())
57+
asic_list.append(asic_id)
58+
return asic_list
59+
60+
61+
#
62+
# Syntax: fabric_module_set_admin_status <chassis_module_name> <'up'/'down'>
63+
#
64+
def fabric_module_set_admin_status(db, chassis_module_name, state):
65+
chassisdb = db.db
66+
chassisdb.connect("CHASSIS_STATE_DB")
67+
asic_list = get_asic_list_from_db(chassisdb, chassis_module_name)
68+
69+
if len(asic_list) == 0:
70+
return
71+
72+
if state == "down":
73+
for asic in asic_list:
74+
click.echo("Stop swss@{} and peer services".format(asic))
75+
clicommon.run_command('sudo systemctl stop swss@{}.service'.format(asic))
76+
77+
is_active = subprocess.call(["systemctl", "is-active", "--quiet", "swss@{}.service".format(asic)])
78+
79+
if is_active == 0: # zero active, non-zero, inactive
80+
click.echo("Stop swss@{} and peer services failed".format(asic))
81+
return
82+
83+
click.echo("Delete related CAHSSIS_FABRIC_ASIC_TABLE entries")
84+
85+
for asic in asic_list:
86+
chassisdb.delete("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic" + str(asic))
87+
88+
# Start the services in case of the users just execute issue command "systemctl stop swss@/syncd@"
89+
# without bring down the hardware
90+
for asic in asic_list:
91+
# To address systemd service restart limit by resetting the count
92+
clicommon.run_command('sudo systemctl reset-failed swss@{}.service'.format(asic))
93+
click.echo("Start swss@{} and peer services".format(asic))
94+
clicommon.run_command('sudo systemctl start swss@{}.service'.format(asic))
95+
elif state == "up":
96+
for asic in asic_list:
97+
click.echo("Start swss@{} and peer services".format(asic))
98+
clicommon.run_command('sudo systemctl start swss@{}.service'.format(asic))
99+
20100
#
21101
# 'shutdown' subcommand ('config chassis_modules shutdown ...')
22102
#
@@ -33,8 +113,17 @@ def shutdown_chassis_module(db, chassis_module_name):
33113
not chassis_module_name.startswith("FABRIC-CARD"):
34114
ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD' or 'FABRIC-CARD'")
35115

116+
# To avoid duplicate operation
117+
if get_config_module_state(db, chassis_module_name) == 'down':
118+
click.echo("Module {} is already in down state".format(chassis_module_name))
119+
return
120+
121+
click.echo("Shutting down chassis module {}".format(chassis_module_name))
36122
fvs = {'admin_status': 'down'}
37123
config_db.set_entry('CHASSIS_MODULE', chassis_module_name, fvs)
124+
if chassis_module_name.startswith("FABRIC-CARD"):
125+
if not check_config_module_state_with_timeout(ctx, db, chassis_module_name, 'down'):
126+
fabric_module_set_admin_status(db, chassis_module_name, 'down')
38127

39128
#
40129
# 'startup' subcommand ('config chassis_modules startup ...')
@@ -45,5 +134,15 @@ def shutdown_chassis_module(db, chassis_module_name):
45134
def startup_chassis_module(db, chassis_module_name):
46135
"""Chassis-module startup of module"""
47136
config_db = db.cfgdb
137+
ctx = click.get_current_context()
138+
139+
# To avoid duplicate operation
140+
if get_config_module_state(db, chassis_module_name) == 'up':
141+
click.echo("Module {} is already set to up state".format(chassis_module_name))
142+
return
48143

144+
click.echo("Starting up chassis module {}".format(chassis_module_name))
49145
config_db.set_entry('CHASSIS_MODULE', chassis_module_name, None)
146+
if chassis_module_name.startswith("FABRIC-CARD"):
147+
if not check_config_module_state_with_timeout(ctx, db, chassis_module_name, 'up'):
148+
fabric_module_set_admin_status(db, chassis_module_name, 'up')

‎tests/chassis_modules_test.py

100644100755
+113
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import tests.mock_tables.dbconnector
88
from utilities_common.db import Db
99
from .utils import get_result_and_return_code
10+
from unittest import mock
11+
sys.modules['clicommon'] = mock.Mock()
1012

1113
show_linecard0_shutdown_output="""\
1214
LINE-CARD0 line-card 1 Empty down LC1000101
@@ -15,6 +17,15 @@
1517
show_linecard0_startup_output="""\
1618
LINE-CARD0 line-card 1 Empty up LC1000101
1719
"""
20+
21+
show_fabriccard0_shutdown_output = """\
22+
FABRIC-CARD0 fabric-card 17 Online down FC1000101
23+
"""
24+
25+
show_fabriccard0_startup_output = """\
26+
FABRIC-CARD0 fabric-card 17 Online up FC1000101
27+
"""
28+
1829
header_lines = 2
1930
warning_lines = 0
2031

@@ -113,6 +124,11 @@
113124
Linecard4|Asic2|PortChannel0001 2 22 Linecard4|Asic2|Ethernet29, Linecard4|Asic2|Ethernet30
114125
"""
115126

127+
128+
def mock_run_command_side_effect(*args, **kwargs):
129+
return '', 0
130+
131+
116132
class TestChassisModules(object):
117133
@classmethod
118134
def setup_class(cls):
@@ -186,6 +202,47 @@ def test_config_shutdown_module(self):
186202
#db.cfgdb.set_entry("CHASSIS_MODULE", "LINE-CARD0", { "admin_status" : "down" })
187203
#db.get_data("CHASSIS_MODULE", "LINE-CARD0")
188204

205+
def test_config_shutdown_module_fabric(self):
206+
with mock.patch("utilities_common.cli.run_command",
207+
mock.MagicMock(side_effect=mock_run_command_side_effect)) as mock_run_command:
208+
runner = CliRunner()
209+
db = Db()
210+
211+
chassisdb = db.db
212+
chassisdb.connect("CHASSIS_STATE_DB")
213+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "asic_id_in_module", "0")
214+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "asic_pci_address", "nokia-bdb:4:0")
215+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "name", "FABRIC-CARD0")
216+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "asic_id_in_module", "1")
217+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "asic_pci_address", "nokia-bdb:4:1")
218+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "name", "FABRIC-CARD0")
219+
chassisdb.close("CHASSIS_STATE_DB")
220+
221+
result = runner.invoke(config.config.commands["chassis"].commands["modules"].commands["shutdown"],
222+
["FABRIC-CARD0"], obj=db)
223+
print(result.exit_code)
224+
print(result.output)
225+
assert result.exit_code == 0
226+
227+
result = runner.invoke(show.cli.commands["chassis"].commands["modules"].commands["status"],
228+
["FABRIC-CARD0"], obj=db)
229+
print(result.exit_code)
230+
print(result.output)
231+
result_lines = result.output.strip('\n').split('\n')
232+
assert result.exit_code == 0
233+
header_lines = 2
234+
result_out = " ".join((result_lines[header_lines]).split())
235+
assert result_out.strip('\n') == show_fabriccard0_shutdown_output.strip('\n')
236+
237+
fvs = {'admin_status': 'down'}
238+
db.cfgdb.set_entry('CHASSIS_MODULE', "FABRIC-CARD0", fvs)
239+
result = runner.invoke(config.config.commands["chassis"].commands["modules"].commands["shutdown"],
240+
["FABRIC-CARD0"], obj=db)
241+
print(result.exit_code)
242+
print(result.output)
243+
assert result.exit_code == 0
244+
assert mock_run_command.call_count == 6
245+
189246
def test_config_startup_module(self):
190247
runner = CliRunner()
191248
db = Db()
@@ -202,6 +259,62 @@ def test_config_startup_module(self):
202259
result_out = " ".join((result_lines[header_lines]).split())
203260
assert result_out.strip('\n') == show_linecard0_startup_output.strip('\n')
204261

262+
def test_config_startup_module_fabric(self):
263+
with mock.patch("utilities_common.cli.run_command",
264+
mock.MagicMock(side_effect=mock_run_command_side_effect)) as mock_run_command:
265+
runner = CliRunner()
266+
db = Db()
267+
268+
chassisdb = db.db
269+
chassisdb.connect("CHASSIS_STATE_DB")
270+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "asic_id_in_module", "0")
271+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "asic_pci_address", "nokia-bdb:4:0")
272+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic6", "name", "FABRIC-CARD0")
273+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "asic_id_in_module", "1")
274+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "asic_pci_address", "nokia-bdb:4:1")
275+
chassisdb.set("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE|asic7", "name", "FABRIC-CARD0")
276+
chassisdb.close("CHASSIS_STATE_DB")
277+
278+
# FC is down and doing startup
279+
fvs = {'admin_status': 'down'}
280+
db.cfgdb.set_entry('CHASSIS_MODULE', "FABRIC-CARD0", fvs)
281+
282+
result = runner.invoke(config.config.commands["chassis"].commands["modules"].commands["startup"],
283+
["FABRIC-CARD0"], obj=db)
284+
print(result.exit_code)
285+
print(result.output)
286+
assert result.exit_code == 0
287+
288+
result = runner.invoke(show.cli.commands["chassis"].commands["modules"].commands["status"],
289+
["FABRIC-CARD0"], obj=db)
290+
print(result.exit_code)
291+
print(result.output)
292+
result_lines = result.output.strip('\n').split('\n')
293+
assert result.exit_code == 0
294+
result_out = " ".join((result_lines[header_lines]).split())
295+
assert result_out.strip('\n') == show_fabriccard0_startup_output.strip('\n')
296+
assert mock_run_command.call_count == 2
297+
298+
# FC is up and doing startup
299+
fvs = {'admin_status': 'up'}
300+
db.cfgdb.set_entry('CHASSIS_MODULE', "FABRIC-CARD0", fvs)
301+
302+
result = runner.invoke(config.config.commands["chassis"].commands["modules"].commands["startup"],
303+
["FABRIC-CARD0"], obj=db)
304+
print(result.exit_code)
305+
print(result.output)
306+
assert result.exit_code == 0
307+
308+
result = runner.invoke(show.cli.commands["chassis"].commands["modules"].commands["status"],
309+
["FABRIC-CARD0"], obj=db)
310+
print(result.exit_code)
311+
print(result.output)
312+
result_lines = result.output.strip('\n').split('\n')
313+
assert result.exit_code == 0
314+
result_out = " ".join((result_lines[header_lines]).split())
315+
assert result_out.strip('\n') == show_fabriccard0_startup_output.strip('\n')
316+
assert mock_run_command.call_count == 2
317+
205318
def test_config_incorrect_module(self):
206319
runner = CliRunner()
207320
db = Db()

0 commit comments

Comments
 (0)