From 34822e2dcdd016f1c8174d465481a4f1f33ca627 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Wed, 17 Apr 2019 22:07:32 +0000 Subject: [PATCH 01/14] Write fields to FLEX_COUNTER_GROUP_TABLE at the construction of a BufferOrch object: "POLL_INTERVAL" "BUFFER_POLL_PLUGIN_LIST" "STATS_MODE" Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 47 +++++++++++++++++++++++++++++++++++++++- orchagent/bufferorch.h | 11 ++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index 9de842142d..8cb6cf9c2e 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -14,6 +14,8 @@ extern sai_buffer_api_t *sai_buffer_api; extern PortsOrch *gPortsOrch; extern sai_object_id_t gSwitchId; +#define BUFFER_POOL_WATERMARK_FLEX_STAT_COUNTER_POLL_MSECS "10000" + using namespace std; type_map BufferOrch::m_buffer_type_maps = { @@ -25,11 +27,18 @@ type_map BufferOrch::m_buffer_type_maps = { {CFG_BUFFER_PORT_EGRESS_PROFILE_LIST_NAME, new object_map()} }; -BufferOrch::BufferOrch(DBConnector *db, vector &tableNames) : Orch(db, tableNames) +BufferOrch::BufferOrch(DBConnector *db, vector &tableNames) : + Orch(db, tableNames), + m_flexCounterDb(new DBConnector(FLEX_COUNTER_DB, DBConnector::DEFAULT_UNIXSOCKET, 0)), + m_flexCounterTable(new ProducerTable(m_flexCounterDb.get(), FLEX_COUNTER_TABLE)), + m_flexCounterGroupTable(new ProducerTable(m_flexCounterDb.get(), FLEX_COUNTER_GROUP_TABLE)), + m_countersDb(new DBConnector(COUNTERS_DB, DBConnector::DEFAULT_UNIXSOCKET, 0)), + m_bufferPoolTable(new Table(m_countersDb.get(), COUNTERS_BUFFER_POOL_NAME_MAP)) { SWSS_LOG_ENTER(); initTableHandlers(); initBufferReadyLists(db); + initFlexCounterGroupTable(); }; void BufferOrch::initTableHandlers() @@ -82,6 +91,42 @@ void BufferOrch::initBufferReadyList(Table& table) } } +void BufferOrch::initFlexCounterGroupTable(void) +{ + string bufferPoolWmPluginName = "watermark_bufferpool.lua"; + + try + { + string bufferPoolLuaScript = swss::loadLuaScript(bufferPoolWmPluginName); + string bufferPoolWmSha = swss::loadRedisScript(m_countersDb.get(), bufferPoolLuaScript); + + vector fvTuples; + fvTuples.emplace_back(BUFFER_POOL_PLUGIN_FIELD, bufferPoolWmSha); + fvTuples.emplace_back(POLL_INTERVAL_FIELD, BUFFER_POOL_WATERMARK_FLEX_STAT_COUNTER_POLL_MSECS); + + string statsMode = STATS_MODE_READ_AND_CLEAR; + // Some platforms do not support buffer pool watermark clear operation + // Need the SAI API support to query the capability + // Before this capability is in place, we feed the platform info into the orchagent + // daemon and check against it + char *device = getenv("device"); + if (device) + { + SWSS_LOG_ERROR("initFlexCounterGroupTable: device: %s", device); + } + if (device && strstr(device, "7050")) + { + statsMode = STATS_MODE_READ; + } + fvTuples.emplace_back(STATS_MODE_FIELD, statsMode); + m_flexCounterGroupTable->set(BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP, fvTuples); + } + catch (...) + { + SWSS_LOG_WARN("Buffer pool watermark lua script and/or flex counter group not set successfully"); + } +} + bool BufferOrch::isPortReady(const std::string& port_name) const { SWSS_LOG_ENTER(); diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index f5811fdbd2..c19b5b6342 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -6,6 +6,9 @@ #include #include "orch.h" #include "portsorch.h" +#include "redisapi.h" + +#define BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP "BUFFER_POOL_WATERMARK_STAT_COUNTER" const string buffer_size_field_name = "size"; const string buffer_pool_type_field_name = "type"; @@ -40,6 +43,7 @@ class BufferOrch : public Orch void initTableHandlers(); void initBufferReadyLists(DBConnector *db); void initBufferReadyList(Table& table); + void initFlexCounterGroupTable(void); task_process_status processBufferPool(Consumer &consumer); task_process_status processBufferProfile(Consumer &consumer); task_process_status processQueue(Consumer &consumer); @@ -50,6 +54,13 @@ class BufferOrch : public Orch buffer_table_handler_map m_bufferHandlerMap; std::unordered_map m_ready_list; std::unordered_map> m_port_ready_list_ref; + + unique_ptr m_flexCounterDb = nullptr; + unique_ptr m_flexCounterGroupTable = nullptr; + unique_ptr m_flexCounterTable = nullptr; + + unique_ptr m_countersDb = nullptr; + unique_ptr m_bufferPoolTable = nullptr; }; #endif /* SWSS_BUFFORCH_H */ From 7f2050fd96918bf2f293bbfcaf1f9e247d294624 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Thu, 18 Apr 2019 00:06:53 +0000 Subject: [PATCH 02/14] Update buffer pool name to oid mapping in COUNTERS_DB upon the set and del of its oid Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 9 ++++++++- orchagent/bufferorch.h | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index 8cb6cf9c2e..ab8e0b06c7 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -33,7 +33,7 @@ BufferOrch::BufferOrch(DBConnector *db, vector &tableNames) : m_flexCounterTable(new ProducerTable(m_flexCounterDb.get(), FLEX_COUNTER_TABLE)), m_flexCounterGroupTable(new ProducerTable(m_flexCounterDb.get(), FLEX_COUNTER_GROUP_TABLE)), m_countersDb(new DBConnector(COUNTERS_DB, DBConnector::DEFAULT_UNIXSOCKET, 0)), - m_bufferPoolTable(new Table(m_countersDb.get(), COUNTERS_BUFFER_POOL_NAME_MAP)) + m_countersDbRedisClient(m_countersDb.get()) { SWSS_LOG_ENTER(); initTableHandlers(); @@ -254,6 +254,12 @@ task_process_status BufferOrch::processBufferPool(Consumer &consumer) } (*(m_buffer_type_maps[map_type_name]))[object_name] = sai_object; SWSS_LOG_NOTICE("Created buffer pool %s with type %s", object_name.c_str(), map_type_name.c_str()); + // Here we take the PFC watchdog approach to update the COUNTERS_DB metadata (e.g., PFC_WD_DETECTION_TIME per queue) + // at initialization (creation and registration phase) + // Specifically, we push the buffer pool name to oid mapping upon the creation of the oid + // In pg and queue case, this mapping installment is deferred to FlexCounterOrch at a reception of field + // "FLEX_COUNTER_STATUS" + m_countersDbRedisClient.hset(COUNTERS_BUFFER_POOL_NAME_MAP, object_name, sai_serialize_object_id(sai_object)); } } else if (op == DEL_COMMAND) @@ -267,6 +273,7 @@ task_process_status BufferOrch::processBufferPool(Consumer &consumer) SWSS_LOG_NOTICE("Removed buffer pool %s with type %s", object_name.c_str(), map_type_name.c_str()); auto it_to_delete = (m_buffer_type_maps[map_type_name])->find(object_name); (m_buffer_type_maps[map_type_name])->erase(it_to_delete); + m_countersDbRedisClient.hdel(COUNTERS_BUFFER_POOL_NAME_MAP, object_name); } else { diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index c19b5b6342..5e552b62b9 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -60,7 +60,7 @@ class BufferOrch : public Orch unique_ptr m_flexCounterTable = nullptr; unique_ptr m_countersDb = nullptr; - unique_ptr
m_bufferPoolTable = nullptr; + RedisClient m_countersDbRedisClient; }; #endif /* SWSS_BUFFORCH_H */ From 959c38f3bc468ce458f49b093ab3b65bd35e5bdc Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Fri, 19 Apr 2019 01:00:32 +0000 Subject: [PATCH 03/14] Push buffer pool watermark COUNTER_ID_LIST to FLEX_COUNTER_TABLE Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 47 +++++++++++++++++++++++++++++++++-- orchagent/bufferorch.h | 5 ++++ orchagent/flexcounterorch.cpp | 26 +++++++++++++++++-- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index ab8e0b06c7..9e8a4d6f8d 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -1,11 +1,13 @@ #include "tokenize.h" - #include "bufferorch.h" #include "logger.h" +#include "sai_serialize.h" #include #include +using namespace std; + extern sai_port_api_t *sai_port_api; extern sai_queue_api_t *sai_queue_api; extern sai_switch_api_t *sai_switch_api; @@ -16,7 +18,11 @@ extern sai_object_id_t gSwitchId; #define BUFFER_POOL_WATERMARK_FLEX_STAT_COUNTER_POLL_MSECS "10000" -using namespace std; + +static const vector bufferPoolWatermarkStatIds = +{ + SAI_BUFFER_POOL_STAT_WATERMARK_BYTES, +}; type_map BufferOrch::m_buffer_type_maps = { {CFG_BUFFER_POOL_TABLE_NAME, new object_map()}, @@ -150,6 +156,43 @@ bool BufferOrch::isPortReady(const std::string& port_name) const return result; } +void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) +{ + // This function will be called in FlexCounterOrch when field:value "FLEX_COUNTER_STATUS":"enable" + // is received on buffer pool watermark key + // Because the SubscriberStateTable listens to the entire keyspace of "FLEX_COUNTER_TABLE", any update + // to "FLEX_COUNTER_TABLE" will cause this tuple to be received again + // To avoid resync the coutner ID list a second time, we introduce a data member variable to mark whether + // this operation has already been done or not yet + if (m_isBufferPoolWatermarkCounterIdListGenerated) + { + SWSS_LOG_ERROR("generateBufferPoolCounterIdList: Buffer pool COUNTER_ID_LIST already generated"); + return; + } + + // Detokenize the SAI watermark stats to a string, separated by comma + string statList; + for (const auto &it : bufferPoolWatermarkStatIds) + { + statList += (sai_serialize_buffer_pool_stat(it) + list_item_delimiter); + } + if (!statList.empty()) + { + statList.pop_back(); + } + SWSS_LOG_ERROR("generateBufferPoolCounterIdList: Buffer pool watermark COUNTER_ID_LIST value: %s", statList.c_str()); + + vector fvTuples; + fvTuples.emplace_back(BUFFER_POOL_COUNTER_ID_LIST, statList); + + // Push buffer pool watermark COUNTER_ID_LIST to FLEX_COUNTER_TABLE on a per buffer pool basis + for (const auto &it : *(m_buffer_type_maps[CFG_BUFFER_POOL_TABLE_NAME])) + { + string key = BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP ":" + sai_serialize_object_id(it.second); + m_flexCounterTable->set(key, fvTuples); + } +} + task_process_status BufferOrch::processBufferPool(Consumer &consumer) { SWSS_LOG_ENTER(); diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index 5e552b62b9..31df8d56df 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -7,6 +7,7 @@ #include "orch.h" #include "portsorch.h" #include "redisapi.h" +#include "redisclient.h" #define BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP "BUFFER_POOL_WATERMARK_STAT_COUNTER" @@ -33,6 +34,8 @@ class BufferOrch : public Orch BufferOrch(DBConnector *db, vector &tableNames); bool isPortReady(const std::string& port_name) const; static type_map m_buffer_type_maps; + void generateBufferPoolWatermarkCounterIdList(void); + private: typedef task_process_status (BufferOrch::*buffer_table_handler)(Consumer& consumer); typedef map buffer_table_handler_map; @@ -61,6 +64,8 @@ class BufferOrch : public Orch unique_ptr m_countersDb = nullptr; RedisClient m_countersDbRedisClient; + + bool m_isBufferPoolWatermarkCounterIdListGenerated = false; }; #endif /* SWSS_BUFFORCH_H */ diff --git a/orchagent/flexcounterorch.cpp b/orchagent/flexcounterorch.cpp index 2935df97ba..1e057bfb5b 100644 --- a/orchagent/flexcounterorch.cpp +++ b/orchagent/flexcounterorch.cpp @@ -6,11 +6,15 @@ #include "redisclient.h" #include "sai_serialize.h" #include "pfcwdorch.h" +#include "bufferorch.h" extern sai_port_api_t *sai_port_api; extern PortsOrch *gPortsOrch; extern IntfsOrch *gIntfsOrch; +extern BufferOrch *gBufferOrch; + +#define BUFFER_POOL_WATERMARK_KEY "BUFFER_POOL_WATERMARK" unordered_map flexCounterGroupMap = { @@ -19,6 +23,7 @@ unordered_map flexCounterGroupMap = {"PFCWD", PFC_WD_FLEX_COUNTER_GROUP}, {"QUEUE_WATERMARK", QUEUE_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP}, {"PG_WATERMARK", PG_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP}, + {BUFFER_POOL_WATERMARK_KEY, BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP}, {"RIF", RIF_STAT_COUNTER_FLEX_COUNTER_GROUP}, }; @@ -76,11 +81,28 @@ void FlexCounterOrch::doTask(Consumer &consumer) } else if(field == FLEX_COUNTER_STATUS_FIELD) { - // Currently the counters are disabled by default - // The queue maps will be generated as soon as counters are enabled + // Currently, the counters are disabled for polling by default + // The queue maps will be generated as soon as counters are enabled for polling + // Counter polling is enabled by pushing the COUNTER_ID_LIST/ATTR_ID_LIST, which contains + // the list of SAI stats/attributes of polling interest, to the FLEX_COUNTER_DB under the + // additional condition that the polling interval at that time is set nonzero positive, + // which is automatically satisfied upon the creation of the orch object that requires + // the syncd flex counter polling service + // This postponement is introduced by design to accelerate the initialization process + // + // generateQueueMap() is called as long as a field "FLEX_COUNTER_STATUS" event is heard, + // regardless of whether the key is "QUEUE" or whether the value is "enable" or "disable" + // This can be because generateQueueMap() installs a fundamental list of queue stats + // that need to be polled. So my doubt here is if queue watermark stats shall be piggybacked + // into the same function as they may not be counted as fundamental gPortsOrch->generateQueueMap(); gPortsOrch->generatePriorityGroupMap(); gIntfsOrch->generateInterfaceMap(); + // Install COUNTER_ID_LIST/ATTR_ID_LIST only when hearing buffer pool watermark enable event + if ((key == BUFFER_POOL_WATERMARK_KEY) && (value == "enable")) + { + gBufferOrch->generateBufferPoolWatermarkCounterIdList(); + } vector fieldValues; fieldValues.emplace_back(FLEX_COUNTER_STATUS_FIELD, value); From 8f080bf56646fc6e7f419a2ccb339c478b83248c Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Sat, 20 Apr 2019 21:16:42 +0000 Subject: [PATCH 04/14] Implement user clear logic to buffer pool watermark Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 16 ++++++++++++---- orchagent/bufferorch.h | 1 + orchagent/watermarkorch.cpp | 29 +++++++++++++++++++++++++---- orchagent/watermarkorch.h | 1 + 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index 9e8a4d6f8d..955ab7e031 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -158,10 +158,10 @@ bool BufferOrch::isPortReady(const std::string& port_name) const void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) { - // This function will be called in FlexCounterOrch when field:value "FLEX_COUNTER_STATUS":"enable" - // is received on buffer pool watermark key - // Because the SubscriberStateTable listens to the entire keyspace of "FLEX_COUNTER_TABLE", any update - // to "FLEX_COUNTER_TABLE" will cause this tuple to be received again + // This function will be called in FlexCounterOrch when field:value tuple "FLEX_COUNTER_STATUS":"enable" + // is received on buffer pool watermark key under table "FLEX_COUNTER_GROUP_TABLE" + // Because the SubscriberStateTable listens to the entire keyspace of "BUFFER_POOL_WATERMARK", any update + // to field value tuples under key "BUFFER_POOL_WATERMARK" will cause this tuple to be heard again // To avoid resync the coutner ID list a second time, we introduce a data member variable to mark whether // this operation has already been done or not yet if (m_isBufferPoolWatermarkCounterIdListGenerated) @@ -193,6 +193,14 @@ void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) } } +const object_map &BufferOrch::getBufferPoolNameOidMap(void) +{ + // In the case different Orches are running in + // different threads, caller may need to grab a read lock + // before calling this function + return *m_buffer_type_maps[CFG_BUFFER_POOL_TABLE_NAME]; +} + task_process_status BufferOrch::processBufferPool(Consumer &consumer) { SWSS_LOG_ENTER(); diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index 31df8d56df..ee141afe4f 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -35,6 +35,7 @@ class BufferOrch : public Orch bool isPortReady(const std::string& port_name) const; static type_map m_buffer_type_maps; void generateBufferPoolWatermarkCounterIdList(void); + const object_map &getBufferPoolNameOidMap(void); private: typedef task_process_status (BufferOrch::*buffer_table_handler)(Consumer& consumer); diff --git a/orchagent/watermarkorch.cpp b/orchagent/watermarkorch.cpp index 5069aaec72..9f758af962 100644 --- a/orchagent/watermarkorch.cpp +++ b/orchagent/watermarkorch.cpp @@ -10,8 +10,10 @@ #define CLEAR_PG_SHARED_REQUEST "PG_SHARED" #define CLEAR_QUEUE_SHARED_UNI_REQUEST "Q_SHARED_UNI" #define CLEAR_QUEUE_SHARED_MULTI_REQUEST "Q_SHARED_MULTI" +#define CLEAR_BUFFER_POOL_REQUEST "BUFFER_POOL" extern PortsOrch *gPortsOrch; +extern BufferOrch *gBufferOrch; WatermarkOrch::WatermarkOrch(DBConnector *db, const string tableName): @@ -123,30 +125,36 @@ void WatermarkOrch::doTask(NotificationConsumer &consumer) return; } - if(data == CLEAR_PG_HEADROOM_REQUEST) + if (data == CLEAR_PG_HEADROOM_REQUEST) { clearSingleWm(table, "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES", m_pg_ids); } - else if(data == CLEAR_PG_SHARED_REQUEST) + else if (data == CLEAR_PG_SHARED_REQUEST) { clearSingleWm(table, "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES", m_pg_ids); } - else if(data == CLEAR_QUEUE_SHARED_UNI_REQUEST) + else if (data == CLEAR_QUEUE_SHARED_UNI_REQUEST) { clearSingleWm(table, "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_unicast_queue_ids); } - else if(data == CLEAR_QUEUE_SHARED_MULTI_REQUEST) + else if (data == CLEAR_QUEUE_SHARED_MULTI_REQUEST) { clearSingleWm(table, "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_multicast_queue_ids); } + else if (data == CLEAR_BUFFER_POOL_REQUEST) + { + clearSingleWm(table, + "SAI_BUFFER_POOL_STAT_WATERMARK_BYTES", + gBufferOrch->getBufferPoolNameOidMap()); + } else { SWSS_LOG_WARN("Unknown watermark clear request data: %s", data.c_str()); @@ -231,3 +239,16 @@ void WatermarkOrch::clearSingleWm(Table *table, string wm_name, vectorset(sai_serialize_object_id(id), vfvt); } } + +void WatermarkOrch::clearSingleWm(Table *table, string wm_name, const object_map &nameOidMap) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_DEBUG("clear WM %s, for %zu obj ids", wm_name.c_str(), nameOidMap.size()); + + vector fvTuples = {{wm_name, "0"}}; + + for (const auto &it : nameOidMap) + { + table->set(sai_serialize_object_id(it.second), fvTuples); + } +} diff --git a/orchagent/watermarkorch.h b/orchagent/watermarkorch.h index b2a9847deb..fd90957dee 100644 --- a/orchagent/watermarkorch.h +++ b/orchagent/watermarkorch.h @@ -22,6 +22,7 @@ class WatermarkOrch : public Orch void init_queue_ids(); void clearSingleWm(Table *table, string wm_name, vector &obj_ids); + void clearSingleWm(Table *table, string wm_name, const object_map &nameOidMap); shared_ptr
getCountersTable(void) { From c2c7ce3118eb2d94b9ffdd2f6f7206f73c58e704 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Sat, 20 Apr 2019 21:28:22 +0000 Subject: [PATCH 05/14] Add periodic clear to buffer pool watermark Signed-off-by: Wenda Ni --- orchagent/watermarkorch.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/orchagent/watermarkorch.cpp b/orchagent/watermarkorch.cpp index 9f758af962..291a55b499 100644 --- a/orchagent/watermarkorch.cpp +++ b/orchagent/watermarkorch.cpp @@ -183,10 +183,16 @@ void WatermarkOrch::doTask(SelectableTimer &timer) m_telemetryTimer->setInterval(intervT); m_telemetryTimer->reset(); - clearSingleWm(m_periodicWatermarkTable.get(), "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES", m_pg_ids); - clearSingleWm(m_periodicWatermarkTable.get(), "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES", m_pg_ids); - clearSingleWm(m_periodicWatermarkTable.get(), "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_unicast_queue_ids); - clearSingleWm(m_periodicWatermarkTable.get(), "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_multicast_queue_ids); + clearSingleWm(m_periodicWatermarkTable.get(), + "SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES", m_pg_ids); + clearSingleWm(m_periodicWatermarkTable.get(), + "SAI_INGRESS_PRIORITY_GROUP_STAT_SHARED_WATERMARK_BYTES", m_pg_ids); + clearSingleWm(m_periodicWatermarkTable.get(), + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_unicast_queue_ids); + clearSingleWm(m_periodicWatermarkTable.get(), + "SAI_QUEUE_STAT_SHARED_WATERMARK_BYTES", m_multicast_queue_ids); + clearSingleWm(m_periodicWatermarkTable.get(), + "SAI_BUFFER_POOL_STAT_WATERMARK_BYTES", gBufferOrch->getBufferPoolNameOidMap()); SWSS_LOG_INFO("Periodic watermark cleared by timer!"); } } From d87ea9793880f2a057648fa5450ab7349aee6ded Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Sun, 21 Apr 2019 05:04:01 +0000 Subject: [PATCH 06/14] Add lua script for watermark_bufferpool Signed-off-by: Wenda Ni --- orchagent/Makefile.am | 3 +- orchagent/watermark_bufferpool.lua | 60 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 orchagent/watermark_bufferpool.lua diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index f8839fb18c..3427fad6d9 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -11,7 +11,8 @@ dist_swss_DATA = \ pfc_detect_nephos.lua \ pfc_restore.lua \ watermark_queue.lua \ - watermark_pg.lua + watermark_pg.lua \ + watermark_bufferpool.lua bin_PROGRAMS = orchagent routeresync orchagent_restart_check diff --git a/orchagent/watermark_bufferpool.lua b/orchagent/watermark_bufferpool.lua new file mode 100644 index 0000000000..bf93bf7fce --- /dev/null +++ b/orchagent/watermark_bufferpool.lua @@ -0,0 +1,60 @@ +-- KEYS - buffer IDs +-- ARGV[1] - counters db index +-- ARGV[2] - counters table name +-- ARGV[3] - poll time interval +-- return nothing for now + +local counters_db = ARGV[1] +local counters_table_name = 'COUNTERS' + +local user_table_name = 'USER_WATERMARKS' +local persistent_table_name = 'PERSISTENT_WATERMARKS' +local periodic_table_name = 'PERIODIC_WATERMARKS' + +local sai_buffer_pool_watermark_stat_name = 'SAI_BUFFER_POOL_STAT_WATERMARK_BYTES' + +local rets = {} + +redis.call('SELECT', counters_db) + +-- Iterate through each buffer pool oid +local n = table.getn(KEYS) +for i = n, 1, -1 do + -- Get new watermark value from COUNTERS + local wm = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) + if wm then + wm = tonumber(wm) + + -- Get last value from *_WATERMARKS + local user_wm_last = redis.call('HGET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) + local persistent_wm_last = redis.call('HGET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) + local periodic_wm_last = redis.call('HGET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) + + -- Set higher value to *_WATERMARKS + if user_wm_last then + user_wm_last = tonumber(user_wm_last) + if wm > user_wm_last then + redis.call('HSET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + else + redis.call('HSET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end + + if persistent_wm_last then + persistent_wm_last = tonumber(persistent_wm_last) + if wm > persistent_wm_last then + redis.call('HSET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + else + redis.call('HSET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end + + if periodic_wm_last then + periodic_wm_last = tonumber(periodic_wm_last) + if wm > periodic_wm_last then + redis.call('HSET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + else + redis.call('HSET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end + end +end + +return rets From 8ff93c88a4af88674a982cf0577da45a20a5a836 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 06:15:33 +0000 Subject: [PATCH 07/14] Fix syntax error in buffer pool watermark lua script Signed-off-by: Wenda Ni --- orchagent/watermark_bufferpool.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orchagent/watermark_bufferpool.lua b/orchagent/watermark_bufferpool.lua index bf93bf7fce..279761e56d 100644 --- a/orchagent/watermark_bufferpool.lua +++ b/orchagent/watermark_bufferpool.lua @@ -35,6 +35,7 @@ for i = n, 1, -1 do user_wm_last = tonumber(user_wm_last) if wm > user_wm_last then redis.call('HSET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end else redis.call('HSET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) end @@ -43,6 +44,7 @@ for i = n, 1, -1 do persistent_wm_last = tonumber(persistent_wm_last) if wm > persistent_wm_last then redis.call('HSET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end else redis.call('HSET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) end @@ -51,6 +53,7 @@ for i = n, 1, -1 do periodic_wm_last = tonumber(periodic_wm_last) if wm > periodic_wm_last then redis.call('HSET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) + end else redis.call('HSET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) end From 84da7c931523d1aab51954be917b7cc120daa056 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 06:19:36 +0000 Subject: [PATCH 08/14] Fix compile error in watermarkorch.cpp Signed-off-by: Wenda Ni --- orchagent/watermarkorch.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/orchagent/watermarkorch.cpp b/orchagent/watermarkorch.cpp index 291a55b499..15dd7160a1 100644 --- a/orchagent/watermarkorch.cpp +++ b/orchagent/watermarkorch.cpp @@ -3,6 +3,7 @@ #include "portsorch.h" #include "notifier.h" #include "converter.h" +#include "bufferorch.h" #define DEFAULT_TELEMETRY_INTERVAL 120 From 73814cb2e5eae7ac03af44deccddd62c1c3e8caa Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 18:42:55 +0000 Subject: [PATCH 09/14] Fix from dut verification Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 2 ++ orchagent/watermark_bufferpool.lua | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index 955ab7e031..a30fbbfdf6 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -191,6 +191,8 @@ void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) string key = BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP ":" + sai_serialize_object_id(it.second); m_flexCounterTable->set(key, fvTuples); } + + m_isBufferPoolWatermarkCounterIdListGenerated = true; } const object_map &BufferOrch::getBufferPoolNameOidMap(void) diff --git a/orchagent/watermark_bufferpool.lua b/orchagent/watermark_bufferpool.lua index 279761e56d..2466ea79c2 100644 --- a/orchagent/watermark_bufferpool.lua +++ b/orchagent/watermark_bufferpool.lua @@ -27,8 +27,6 @@ for i = n, 1, -1 do -- Get last value from *_WATERMARKS local user_wm_last = redis.call('HGET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) - local persistent_wm_last = redis.call('HGET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) - local periodic_wm_last = redis.call('HGET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) -- Set higher value to *_WATERMARKS if user_wm_last then @@ -40,6 +38,7 @@ for i = n, 1, -1 do redis.call('HSET', user_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) end + local persistent_wm_last = redis.call('HGET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) if persistent_wm_last then persistent_wm_last = tonumber(persistent_wm_last) if wm > persistent_wm_last then @@ -49,6 +48,7 @@ for i = n, 1, -1 do redis.call('HSET', persistent_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name, wm) end + local periodic_wm_last = redis.call('HGET', periodic_table_name .. ':' .. KEYS[i], sai_buffer_pool_watermark_stat_name) if periodic_wm_last then periodic_wm_last = tonumber(periodic_wm_last) if wm > periodic_wm_last then From c727057a7239f5c7b42013d0b07ff47ecb1c4461 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 19:14:44 +0000 Subject: [PATCH 10/14] Add 6000 to read only polling mode Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index a30fbbfdf6..bac1b50d71 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -120,7 +120,7 @@ void BufferOrch::initFlexCounterGroupTable(void) { SWSS_LOG_ERROR("initFlexCounterGroupTable: device: %s", device); } - if (device && strstr(device, "7050")) + if (device && (strstr(device, "7050") || strstr(device, "6000"))) { statsMode = STATS_MODE_READ; } From 1ebeec185a309009ccb9d6c88381864b654e291c Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 19:15:57 +0000 Subject: [PATCH 11/14] Touch-up to existing codes Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 2 +- orchagent/bufferorch.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index bac1b50d71..ac43aa0a98 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -475,7 +475,7 @@ task_process_status BufferOrch::processBufferProfile(Consumer &consumer) } /* -Input sample "BUFFER_QUEUE_TABLE:Ethernet4,Ethernet45:10-15" +Input sample "BUFFER_QUEUE|Ethernet4,Ethernet45|10-15" */ task_process_status BufferOrch::processQueue(Consumer &consumer) { diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index ee141afe4f..0a01e6cddb 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -42,7 +42,7 @@ class BufferOrch : public Orch typedef map buffer_table_handler_map; typedef pair buffer_handler_pair; - virtual void doTask() override; + void doTask() override; virtual void doTask(Consumer& consumer); void initTableHandlers(); void initBufferReadyLists(DBConnector *db); From 40ef377ed13abcb9c73ffe0aa34f44d7c2e50e96 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Tue, 23 Apr 2019 19:53:29 +0000 Subject: [PATCH 12/14] Remove debugging symbols Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index ac43aa0a98..4b3b5337b6 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -116,10 +116,6 @@ void BufferOrch::initFlexCounterGroupTable(void) // Before this capability is in place, we feed the platform info into the orchagent // daemon and check against it char *device = getenv("device"); - if (device) - { - SWSS_LOG_ERROR("initFlexCounterGroupTable: device: %s", device); - } if (device && (strstr(device, "7050") || strstr(device, "6000"))) { statsMode = STATS_MODE_READ; @@ -166,7 +162,6 @@ void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) // this operation has already been done or not yet if (m_isBufferPoolWatermarkCounterIdListGenerated) { - SWSS_LOG_ERROR("generateBufferPoolCounterIdList: Buffer pool COUNTER_ID_LIST already generated"); return; } @@ -180,7 +175,6 @@ void BufferOrch::generateBufferPoolWatermarkCounterIdList(void) { statList.pop_back(); } - SWSS_LOG_ERROR("generateBufferPoolCounterIdList: Buffer pool watermark COUNTER_ID_LIST value: %s", statList.c_str()); vector fvTuples; fvTuples.emplace_back(BUFFER_POOL_COUNTER_ID_LIST, statList); From 9ba2fa4fd55c343096f6a0de958444089b1b44f5 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Fri, 31 May 2019 03:41:42 +0000 Subject: [PATCH 13/14] Address comments Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 20 +++++++------------- orchagent/bufferorch.h | 8 ++++---- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index 4b3b5337b6..cd27161fa0 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -110,22 +110,16 @@ void BufferOrch::initFlexCounterGroupTable(void) fvTuples.emplace_back(BUFFER_POOL_PLUGIN_FIELD, bufferPoolWmSha); fvTuples.emplace_back(POLL_INTERVAL_FIELD, BUFFER_POOL_WATERMARK_FLEX_STAT_COUNTER_POLL_MSECS); - string statsMode = STATS_MODE_READ_AND_CLEAR; - // Some platforms do not support buffer pool watermark clear operation - // Need the SAI API support to query the capability - // Before this capability is in place, we feed the platform info into the orchagent - // daemon and check against it - char *device = getenv("device"); - if (device && (strstr(device, "7050") || strstr(device, "6000"))) - { - statsMode = STATS_MODE_READ; - } - fvTuples.emplace_back(STATS_MODE_FIELD, statsMode); + // TODO (work in progress): + // Some platforms do not support buffer pool watermark clear operation on a particular pool + // Invoke the SAI clear_stats API per pool to query the capability from the API call return status + fvTuples.emplace_back(STATS_MODE_FIELD, STATS_MODE_READ_AND_CLEAR); + m_flexCounterGroupTable->set(BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP, fvTuples); } - catch (...) + catch (const exception &e) { - SWSS_LOG_WARN("Buffer pool watermark lua script and/or flex counter group not set successfully"); + SWSS_LOG_WARN("Buffer pool watermark lua script and/or flex counter group not set successfully. Runtime error: %s", e.what()); } } diff --git a/orchagent/bufferorch.h b/orchagent/bufferorch.h index 0a01e6cddb..bbc3448376 100644 --- a/orchagent/bufferorch.h +++ b/orchagent/bufferorch.h @@ -59,11 +59,11 @@ class BufferOrch : public Orch std::unordered_map m_ready_list; std::unordered_map> m_port_ready_list_ref; - unique_ptr m_flexCounterDb = nullptr; - unique_ptr m_flexCounterGroupTable = nullptr; - unique_ptr m_flexCounterTable = nullptr; + unique_ptr m_flexCounterDb; + unique_ptr m_flexCounterGroupTable; + unique_ptr m_flexCounterTable; - unique_ptr m_countersDb = nullptr; + unique_ptr m_countersDb; RedisClient m_countersDbRedisClient; bool m_isBufferPoolWatermarkCounterIdListGenerated = false; From 3d66542ad6416aff9ac9c35851aaaded82625b07 Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Wed, 12 Jun 2019 23:00:49 +0000 Subject: [PATCH 14/14] Address comments Signed-off-by: Wenda Ni --- orchagent/bufferorch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchagent/bufferorch.cpp b/orchagent/bufferorch.cpp index cd27161fa0..e35b4c7f48 100644 --- a/orchagent/bufferorch.cpp +++ b/orchagent/bufferorch.cpp @@ -117,9 +117,9 @@ void BufferOrch::initFlexCounterGroupTable(void) m_flexCounterGroupTable->set(BUFFER_POOL_WATERMARK_STAT_COUNTER_FLEX_COUNTER_GROUP, fvTuples); } - catch (const exception &e) + catch (const runtime_error &e) { - SWSS_LOG_WARN("Buffer pool watermark lua script and/or flex counter group not set successfully. Runtime error: %s", e.what()); + SWSS_LOG_ERROR("Buffer pool watermark lua script and/or flex counter group not set successfully. Runtime error: %s", e.what()); } }