Skip to content

Commit 4f514ad

Browse files
stephenxsstephenxs
authored
Enforce the order when the shared headroom pool is enabled (#2729)
What I did Enforce the order when the shared headroom pool is enabled. This is to backport #2699 to 202211. Signed-off-by: Stephen Sun [email protected] Why I did it The current flow to enable the shared headroom pool Configure the shared headroom pool size or over-subscribe ratio Update lossless buffer profiles with xon == size Calculate and update the shared headroom pool size. In step 2, the lossless buffer profiles have been updated to values as if the shared headroom pool is enabled. However, it is enabled only in step 3, which is inconsistent between steps 2 and 3. Therefore, we open the PR to guarantee the order. The new flow A user configures the shared headroom pool size or over-subscribe ratio The dynamic buffer manager invokes the vendor-specific Lua plugin to calculate the shared headroom pool size This is the step introduced in this PR to guarantee the shared headroom pool will be enabled in advance On Nvidia platform, a non-zero shared headroom pool is returned in this stage if the user configures the over-subscribe ratio If a non-zero shared headroom pool is returned, the dynamic buffer manager pushes the shared headroom pool size to APPL_DB.ingress_lossless_pool and blocks until it has been updated into APPL_STATE_DB.ingress_lossless_pool (which indicates the buffer orchagent finishes handling it) The buffer manager updates the lossless buffer profiles The buffer manager invokes the Lua plugin to calculate the shared headroom pool size. The flow continues as normal. How I verified it Manually test and regression test
1 parent 03238b3 commit 4f514ad

File tree

9 files changed

+110
-8
lines changed

9 files changed

+110
-8
lines changed

‎cfgmgr/buffer_pool_mellanox.lua

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ for name in pairs(profiles) do
324324
size = size + lossypg_reserved
325325
end
326326
if size ~= 0 then
327-
if shp_enabled and shp_size == 0 then
327+
if shp_size == 0 then
328328
local xon = tonumber(redis.call('HGET', name, 'xon'))
329329
local xoff = tonumber(redis.call('HGET', name, 'xoff'))
330330
if xon ~= nil and xoff ~= nil and xon + xoff > size then
@@ -346,6 +346,12 @@ accumulative_occupied_buffer = accumulative_occupied_buffer + lossypg_extra_for_
346346

347347
-- Accumulate sizes for private headrooms
348348
local accumulative_private_headroom = 0
349+
local force_enable_shp = false
350+
if accumulative_xoff > 0 and shp_enabled ~= true then
351+
force_enable_shp = true
352+
shp_size = 655360
353+
shp_enabled = true
354+
end
349355
if shp_enabled then
350356
accumulative_private_headroom = lossless_port_count * private_headroom
351357
accumulative_occupied_buffer = accumulative_occupied_buffer + accumulative_private_headroom
@@ -391,6 +397,9 @@ end
391397

392398
if shp_enabled and shp_size == 0 then
393399
shp_size = math.ceil(accumulative_xoff / over_subscribe_ratio)
400+
if shp_size == 0 then
401+
shp_size = 655360
402+
end
394403
end
395404

396405
local pool_size
@@ -432,6 +441,7 @@ table.insert(result, "debug:mgmt_pool:" .. mgmt_pool_size)
432441
if shp_enabled then
433442
table.insert(result, "debug:accumulative_private_headroom:" .. accumulative_private_headroom)
434443
table.insert(result, "debug:accumulative xoff:" .. accumulative_xoff)
444+
table.insert(result, "debug:force enabled shp:" .. tostring(force_enable_shp))
435445
end
436446
table.insert(result, "debug:accumulative_mgmt_pg:" .. accumulative_management_pg)
437447
table.insert(result, "debug:egress_mirror:" .. accumulative_egress_mirror_overhead)

‎cfgmgr/buffermgrd.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ int main(int argc, char **argv)
189189
WarmStart::initialize("buffermgrd", "swss");
190190
WarmStart::checkWarmStart("buffermgrd", "swss");
191191

192+
DBConnector applStateDb("APPL_STATE_DB", 0);
193+
192194
vector<TableConnector> buffer_table_connectors = {
193195
TableConnector(&cfgDb, CFG_PORT_TABLE_NAME),
194196
TableConnector(&cfgDb, CFG_PORT_CABLE_LEN_TABLE_NAME),
@@ -202,7 +204,7 @@ int main(int argc, char **argv)
202204
TableConnector(&stateDb, STATE_BUFFER_MAXIMUM_VALUE_TABLE),
203205
TableConnector(&stateDb, STATE_PORT_TABLE_NAME)
204206
};
205-
cfgOrchList.emplace_back(new BufferMgrDynamic(&cfgDb, &stateDb, &applDb, buffer_table_connectors, peripherial_table_ptr, zero_profiles_ptr));
207+
cfgOrchList.emplace_back(new BufferMgrDynamic(&cfgDb, &stateDb, &applDb, &applStateDb, buffer_table_connectors, peripherial_table_ptr, zero_profiles_ptr));
206208
}
207209
else if (!pg_lookup_file.empty())
208210
{

‎cfgmgr/buffermgrdyn.cpp

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
using namespace std;
2727
using namespace swss;
2828

29-
BufferMgrDynamic::BufferMgrDynamic(DBConnector *cfgDb, DBConnector *stateDb, DBConnector *applDb, const vector<TableConnector> &tables, shared_ptr<vector<KeyOpFieldsValuesTuple>> gearboxInfo, shared_ptr<vector<KeyOpFieldsValuesTuple>> zeroProfilesInfo) :
29+
BufferMgrDynamic::BufferMgrDynamic(DBConnector *cfgDb, DBConnector *stateDb, DBConnector *applDb, DBConnector *applStateDb, const vector<TableConnector> &tables, shared_ptr<vector<KeyOpFieldsValuesTuple>> gearboxInfo, shared_ptr<vector<KeyOpFieldsValuesTuple>> zeroProfilesInfo) :
3030
Orch(tables),
3131
m_platform(),
3232
m_bufferDirections{BUFFER_INGRESS, BUFFER_EGRESS},
@@ -38,6 +38,7 @@ BufferMgrDynamic::BufferMgrDynamic(DBConnector *cfgDb, DBConnector *stateDb, DBC
3838
m_cfgDefaultLosslessBufferParam(cfgDb, CFG_DEFAULT_LOSSLESS_BUFFER_PARAMETER),
3939
m_cfgDeviceMetaDataTable(cfgDb, CFG_DEVICE_METADATA_TABLE_NAME),
4040
m_applBufferPoolTable(applDb, APP_BUFFER_POOL_TABLE_NAME),
41+
m_applStateBufferPoolTable(applStateDb, APP_BUFFER_POOL_TABLE_NAME),
4142
m_applBufferProfileTable(applDb, APP_BUFFER_PROFILE_TABLE_NAME),
4243
m_applBufferObjectTables{ProducerStateTable(applDb, APP_BUFFER_PG_TABLE_NAME), ProducerStateTable(applDb, APP_BUFFER_QUEUE_TABLE_NAME)},
4344
m_applBufferProfileListTables{ProducerStateTable(applDb, APP_BUFFER_PORT_INGRESS_PROFILE_LIST_NAME), ProducerStateTable(applDb, APP_BUFFER_PORT_EGRESS_PROFILE_LIST_NAME)},
@@ -1960,6 +1961,13 @@ task_process_status BufferMgrDynamic::handleDefaultLossLessBufferParam(KeyOpFiel
19601961
{
19611962
bool isSHPEnabled = isNonZero(m_overSubscribeRatio);
19621963
bool willSHPBeEnabled = isNonZero(newRatio);
1964+
if (m_portInitDone && (!isSHPEnabled) && willSHPBeEnabled)
1965+
{
1966+
if (!isSharedHeadroomPoolEnabledInSai())
1967+
{
1968+
return task_process_status::task_need_retry;
1969+
}
1970+
}
19631971
SWSS_LOG_INFO("Recalculate shared buffer pool size due to over subscribe ratio has been updated from %s to %s",
19641972
m_overSubscribeRatio.c_str(), newRatio.c_str());
19651973
m_overSubscribeRatio = newRatio;
@@ -1968,6 +1976,24 @@ task_process_status BufferMgrDynamic::handleDefaultLossLessBufferParam(KeyOpFiel
19681976

19691977
return task_process_status::task_success;
19701978
}
1979+
bool BufferMgrDynamic::isSharedHeadroomPoolEnabledInSai()
1980+
{
1981+
string xoff;
1982+
recalculateSharedBufferPool();
1983+
if (!isNonZero(m_bufferPoolLookup[INGRESS_LOSSLESS_PG_POOL_NAME].xoff))
1984+
{
1985+
return true;
1986+
}
1987+
m_applBufferPoolTable.flush();
1988+
m_applStateBufferPoolTable.hget(INGRESS_LOSSLESS_PG_POOL_NAME, "xoff", xoff);
1989+
if (!isNonZero(xoff))
1990+
{
1991+
SWSS_LOG_INFO("Shared headroom pool is enabled but has not been applied to SAI, retrying");
1992+
return false;
1993+
}
1994+
1995+
return true;
1996+
}
19711997

19721998
task_process_status BufferMgrDynamic::handleCableLenTable(KeyOpFieldsValuesTuple &tuple)
19731999
{
@@ -2416,6 +2442,14 @@ task_process_status BufferMgrDynamic::handleBufferPoolTable(KeyOpFieldsValuesTup
24162442
{
24172443
bool isSHPEnabledBySize = isNonZero(m_configuredSharedHeadroomPoolSize);
24182444

2445+
if (m_portInitDone && (!isSHPEnabledBySize) && willSHPBeEnabledBySize)
2446+
{
2447+
if (!isSharedHeadroomPoolEnabledInSai())
2448+
{
2449+
return task_process_status::task_need_retry;
2450+
}
2451+
}
2452+
24192453
m_configuredSharedHeadroomPoolSize = newSHPSize;
24202454
refreshSharedHeadroomPool(false, isSHPEnabledBySize != willSHPBeEnabledBySize);
24212455
}

‎cfgmgr/buffermgrdyn.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ typedef std::map<std::string, std::string> gearbox_delay_t;
147147
class BufferMgrDynamic : public Orch
148148
{
149149
public:
150-
BufferMgrDynamic(DBConnector *cfgDb, DBConnector *stateDb, DBConnector *applDb, const std::vector<TableConnector> &tables, std::shared_ptr<std::vector<KeyOpFieldsValuesTuple>> gearboxInfo, std::shared_ptr<std::vector<KeyOpFieldsValuesTuple>> zeroProfilesInfo);
150+
BufferMgrDynamic(DBConnector *cfgDb, DBConnector *stateDb, DBConnector *applDb, DBConnector *applStateDb, const std::vector<TableConnector> &tables, std::shared_ptr<std::vector<KeyOpFieldsValuesTuple>> gearboxInfo, std::shared_ptr<std::vector<KeyOpFieldsValuesTuple>> zeroProfilesInfo);
151151
using Orch::doTask;
152152

153153
private:
@@ -204,6 +204,7 @@ class BufferMgrDynamic : public Orch
204204

205205
// BUFFER_POOL table and cache
206206
ProducerStateTable m_applBufferPoolTable;
207+
Table m_applStateBufferPoolTable;
207208
Table m_stateBufferPoolTable;
208209
buffer_pool_lookup_t m_bufferPoolLookup;
209210

@@ -294,6 +295,7 @@ class BufferMgrDynamic : public Orch
294295
task_process_status allocateProfile(const std::string &speed, const std::string &cable, const std::string &mtu, const std::string &threshold, const std::string &gearbox_model, long lane_count, std::string &profile_name);
295296
void releaseProfile(const std::string &profile_name);
296297
bool isHeadroomResourceValid(const std::string &port, const buffer_profile_t &profile, const std::string &new_pg);
298+
bool isSharedHeadroomPoolEnabledInSai();
297299
void refreshSharedHeadroomPool(bool enable_state_updated_by_ratio, bool enable_state_updated_by_size);
298300
task_process_status checkBufferProfileDirection(const std::string &profiles, buffer_direction_t dir);
299301
std::string constructZeroProfileListFromNormalProfileList(const std::string &normalProfileList, const std::string &port);

‎orchagent/bufferorch.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
323323
string map_type_name = APP_BUFFER_POOL_TABLE_NAME;
324324
string object_name = kfvKey(tuple);
325325
string op = kfvOp(tuple);
326+
string xoff;
326327

327328
SWSS_LOG_DEBUG("object name:%s", object_name.c_str());
328329
if (m_buffer_type_maps[map_type_name]->find(object_name) != m_buffer_type_maps[map_type_name]->end())
@@ -417,6 +418,7 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
417418
attr.value.u64 = (uint64_t)stoul(value);
418419
attr.id = SAI_BUFFER_POOL_ATTR_XOFF_SIZE;
419420
attribs.push_back(attr);
421+
xoff = value;
420422
}
421423
else
422424
{
@@ -469,6 +471,15 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
469471
// "FLEX_COUNTER_STATUS"
470472
m_countersDb->hset(COUNTERS_BUFFER_POOL_NAME_MAP, object_name, sai_serialize_object_id(sai_object));
471473
}
474+
475+
// Only publish the result when shared headroom pool is enabled and it has been successfully applied to SAI
476+
if (!xoff.empty())
477+
{
478+
vector<FieldValueTuple> fvs;
479+
fvs.emplace_back("xoff", xoff);
480+
SWSS_LOG_INFO("Publishing the result after applying the shared headroom pool size %s to SAI", xoff.c_str());
481+
m_publisher.publish(APP_BUFFER_POOL_TABLE_NAME, object_name, fvs, ReturnCode(SAI_STATUS_SUCCESS), true);
482+
}
472483
}
473484
else if (op == DEL_COMMAND)
474485
{
@@ -499,6 +510,9 @@ task_process_status BufferOrch::processBufferPool(KeyOpFieldsValuesTuple &tuple)
499510
auto it_to_delete = (m_buffer_type_maps[map_type_name])->find(object_name);
500511
(m_buffer_type_maps[map_type_name])->erase(it_to_delete);
501512
m_countersDb->hdel(COUNTERS_BUFFER_POOL_NAME_MAP, object_name);
513+
514+
vector<FieldValueTuple> fvs;
515+
m_publisher.publish(APP_BUFFER_POOL_TABLE_NAME, object_name, fvs, ReturnCode(SAI_STATUS_SUCCESS), true);
502516
}
503517
else
504518
{

‎tests/mock_tests/Makefile.am

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ tests_intfmgrd_INCLUDES = $(tests_INCLUDES) -I$(top_srcdir)/cfgmgr -I$(top_srcdi
172172
tests_intfmgrd_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(CFLAGS_SAI)
173173
tests_intfmgrd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(CFLAGS_SAI) $(tests_intfmgrd_INCLUDES)
174174
tests_intfmgrd_LDADD = $(LDADD_GTEST) $(LDADD_SAI) -lnl-genl-3 -lhiredis -lhiredis \
175-
-lswsscommon -lswsscommon -lgtest -lgtest_main -lzmq -lnl-3 -lnl-route-3 -lpthread
175+
-lswsscommon -lswsscommon -lgtest -lgtest_main -lzmq -lnl-3 -lnl-route-3 -lpthread -lgmock -lgmock_main
176176

177177
## fpmsyncd unit tests
178178

‎tests/mock_tests/buffermgrdyn_ut.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ namespace buffermgrdyn_test
2222
shared_ptr<swss::DBConnector> m_app_db = make_shared<swss::DBConnector>("APPL_DB", 0);
2323
shared_ptr<swss::DBConnector> m_config_db = make_shared<swss::DBConnector>("CONFIG_DB", 0);
2424
shared_ptr<swss::DBConnector> m_state_db = make_shared<swss::DBConnector>("STATE_DB", 0);
25+
shared_ptr<swss::DBConnector> m_app_state_db = make_shared<swss::DBConnector>("APPL_STATE_DB", 0);
2526

2627
BufferMgrDynamic *m_dynamicBuffer;
2728
SelectableTimer m_selectableTable(timespec({ .tv_sec = BUFFERMGR_TIMER_PERIOD, .tv_nsec = 0 }), 0);
@@ -180,7 +181,7 @@ namespace buffermgrdyn_test
180181
TableConnector(m_state_db.get(), STATE_PORT_TABLE_NAME)
181182
};
182183

183-
m_dynamicBuffer = new BufferMgrDynamic(m_config_db.get(), m_state_db.get(), m_app_db.get(), buffer_table_connectors, nullptr, zero_profile);
184+
m_dynamicBuffer = new BufferMgrDynamic(m_config_db.get(), m_state_db.get(), m_app_db.get(), m_app_state_db.get(), buffer_table_connectors, nullptr, zero_profile);
184185
}
185186

186187
void InitPort(const string &port="Ethernet0", const string &admin_status="up")

‎tests/mock_tests/bufferorch_ut.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
#include "ut_helper.h"
88
#include "mock_orchagent_main.h"
99
#include "mock_table.h"
10+
#include "mock_response_publisher.h"
1011

1112
extern string gMySwitchType;
1213

14+
extern std::unique_ptr<MockResponsePublisher> gMockResponsePublisher;
1315

1416
namespace bufferorch_test
1517
{
@@ -19,6 +21,7 @@ namespace bufferorch_test
1921
sai_port_api_t *pold_sai_port_api;
2022

2123
shared_ptr<swss::DBConnector> m_app_db;
24+
shared_ptr<swss::DBConnector> m_app_state_db;
2225
shared_ptr<swss::DBConnector> m_config_db;
2326
shared_ptr<swss::DBConnector> m_state_db;
2427
shared_ptr<swss::DBConnector> m_chassis_app_db;
@@ -113,6 +116,7 @@ namespace bufferorch_test
113116
m_app_db = make_shared<swss::DBConnector>("APPL_DB", 0);
114117
m_config_db = make_shared<swss::DBConnector>("CONFIG_DB", 0);
115118
m_state_db = make_shared<swss::DBConnector>("STATE_DB", 0);
119+
m_app_state_db = make_shared<swss::DBConnector>("APPL_STATE_DB", 0);
116120
if(gMySwitchType == "voq")
117121
m_chassis_app_db = make_shared<swss::DBConnector>("CHASSIS_APP_DB", 0);
118122

@@ -317,6 +321,24 @@ namespace bufferorch_test
317321
}
318322
};
319323

324+
TEST_F(BufferOrchTest, BufferOrchTestSharedHeadroomPool)
325+
{
326+
gMockResponsePublisher = std::make_unique<MockResponsePublisher>();
327+
328+
Table bufferPoolTable = Table(m_app_db.get(), APP_BUFFER_POOL_TABLE_NAME);
329+
Table bufferPoolStateTable = Table(m_app_state_db.get(), APP_BUFFER_POOL_TABLE_NAME);
330+
331+
bufferPoolTable.set("ingress_lossless_pool",
332+
{
333+
{"xoff", "10240"}
334+
});
335+
gBufferOrch->addExistingData(&bufferPoolTable);
336+
EXPECT_CALL(*gMockResponsePublisher, publish(APP_BUFFER_POOL_TABLE_NAME, "ingress_lossless_pool", std::vector<FieldValueTuple>{{"xoff", "10240"}}, ReturnCode(SAI_STATUS_SUCCESS), true)).Times(1);
337+
static_cast<Orch *>(gBufferOrch)->doTask();
338+
339+
gMockResponsePublisher.reset();
340+
}
341+
320342
TEST_F(BufferOrchTest, BufferOrchTestBufferPgReferencingObjRemoveThenAdd)
321343
{
322344
vector<string> ts;

‎tests/mock_tests/fake_response_publisher.cpp

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,36 @@
22
#include <vector>
33

44
#include "response_publisher.h"
5+
#include "mock_response_publisher.h"
6+
7+
/* This mock plugs into this fake response publisher implementation
8+
* when needed to test code that uses response publisher. */
9+
std::unique_ptr<MockResponsePublisher> gMockResponsePublisher;
510

611
ResponsePublisher::ResponsePublisher() : m_db("APPL_STATE_DB", 0) {}
712

813
void ResponsePublisher::publish(
914
const std::string& table, const std::string& key,
1015
const std::vector<swss::FieldValueTuple>& intent_attrs,
1116
const ReturnCode& status,
12-
const std::vector<swss::FieldValueTuple>& state_attrs, bool replace) {}
17+
const std::vector<swss::FieldValueTuple>& state_attrs, bool replace)
18+
{
19+
if (gMockResponsePublisher)
20+
{
21+
gMockResponsePublisher->publish(table, key, intent_attrs, status, state_attrs, replace);
22+
}
23+
}
1324

1425
void ResponsePublisher::publish(
1526
const std::string& table, const std::string& key,
1627
const std::vector<swss::FieldValueTuple>& intent_attrs,
17-
const ReturnCode& status, bool replace) {}
28+
const ReturnCode& status, bool replace)
29+
{
30+
if (gMockResponsePublisher)
31+
{
32+
gMockResponsePublisher->publish(table, key, intent_attrs, status, replace);
33+
}
34+
}
1835

1936
void ResponsePublisher::writeToDB(
2037
const std::string& table, const std::string& key,

0 commit comments

Comments
 (0)