[pfcwd] Remove APPL_DB queue in-storm status at pfcwd config removal and big red switch enable

orchagent/countercheckorch.cpp

@@ -56,7 +56,8 @@ void CounterCheckOrch::mcCounterCheck()
     {
         auto oid = i.first;
         auto mcCounters = i.second;
-        uint8_t pfcMask = 0;
+        uint8_t dummy = 0;
+        uint8_t pfcMaskCfg = 0;
 
         Port port;
         if (!gPortsOrch->getPort(oid, port))
@@ -67,15 +68,15 @@ void CounterCheckOrch::mcCounterCheck()
 
         auto newMcCounters = getQueueMcCounters(port);
 
-        if (!gPortsOrch->getPortPfc(port.m_port_id, &pfcMask))
+        if (!gPortsOrch->getPortPfc(port.m_port_id, dummy, pfcMaskCfg))
         {
             SWSS_LOG_ERROR("Failed to get PFC mask on port %s", port.m_alias.c_str());
             continue;
         }
 
         for (size_t prio = 0; prio != mcCounters.size(); prio++)
         {
-            bool isLossy = ((1 << prio) & pfcMask) == 0;
+            bool isLossy = ((1 << prio) & pfcMaskCfg) == 0;
             if (newMcCounters[prio] == numeric_limits<uint64_t>::max())
             {
                 SWSS_LOG_WARN("Could not retreive MC counters on queue %zu port %s",
@@ -104,7 +105,8 @@ void CounterCheckOrch::pfcFrameCounterCheck()
         auto oid = i.first;
         auto counters = i.second;
         auto newCounters = getPfcFrameCounters(oid);
-        uint8_t pfcMask = 0;
+        uint8_t dummy = 0;
+        uint8_t pfcMaskCfg = 0;
 
         Port port;
         if (!gPortsOrch->getPort(oid, port))
@@ -113,15 +115,15 @@ void CounterCheckOrch::pfcFrameCounterCheck()
             continue;
         }
 
-        if (!gPortsOrch->getPortPfc(port.m_port_id, &pfcMask))
+        if (!gPortsOrch->getPortPfc(port.m_port_id, dummy, pfcMaskCfg))
         {
             SWSS_LOG_ERROR("Failed to get PFC mask on port %s", port.m_alias.c_str());
             continue;
         }
 
         for (size_t prio = 0; prio != counters.size(); prio++)
         {
-            bool isLossy = ((1 << prio) & pfcMask) == 0;
+            bool isLossy = ((1 << prio) & pfcMaskCfg) == 0;
             if (newCounters[prio] == numeric_limits<uint64_t>::max())
             {
                 SWSS_LOG_WARN("Could not retreive PFC frame count on queue %zu port %s",

orchagent/pfcactionhandler.cpp

@@ -238,6 +238,7 @@ PfcWdAclHandler::PfcWdAclHandler(sai_object_id_t port, sai_object_id_t queue,
     {
         // First time of handling PFC for this queue, create ACL table, and bind
         createPfcAclTable(port, m_strIngressTable, true);
+        // Create acl rule in bound acl table
         shared_ptr<AclRulePfcwd> newRule = make_shared<AclRulePfcwd>(gAclOrch, m_strRule, m_strIngressTable, table_type);
         createPfcAclRule(newRule, queueId, m_strIngressTable, port);
     }
@@ -393,16 +394,17 @@ PfcWdLossyHandler::PfcWdLossyHandler(sai_object_id_t port, sai_object_id_t queue
 {
     SWSS_LOG_ENTER();
 
-    uint8_t pfcMask = 0;
+    uint8_t pfcMaskWdCfg = 0;
+    uint8_t dummy = 0;
 
-    if (!gPortsOrch->getPortPfc(port, &pfcMask))
+    if (!gPortsOrch->getPortPfc(port, pfcMaskWdCfg, dummy))
     {
         SWSS_LOG_ERROR("Failed to get PFC mask on port 0x%" PRIx64, port);
     }
 
-    pfcMask = static_cast<uint8_t>(pfcMask & ~(1 << queueId));
+    pfcMaskWdCfg = static_cast<uint8_t>(pfcMaskWdCfg & ~(1 << queueId));
 
-    if (!gPortsOrch->setPortPfc(port, pfcMask))
+    if (!gPortsOrch->setPortPfcStatus(port, pfcMaskWdCfg))
     {
         SWSS_LOG_ERROR("Failed to set PFC mask on port 0x%" PRIx64, port);
     }
@@ -412,16 +414,18 @@ PfcWdLossyHandler::~PfcWdLossyHandler(void)
 {
     SWSS_LOG_ENTER();
 
-    uint8_t pfcMask = 0;
+    uint8_t pfcMaskWdCfg = 0;
+    uint8_t pfcMaskUserCfg = 0;
 
-    if (!gPortsOrch->getPortPfc(getPort(), &pfcMask))
+    if (!gPortsOrch->getPortPfc(getPort(), pfcMaskWdCfg, pfcMaskUserCfg))
     {
         SWSS_LOG_ERROR("Failed to get PFC mask on port 0x%" PRIx64, getPort());
     }
 
-    pfcMask = static_cast<uint8_t>(pfcMask | (1 << getQueueId()));
+    // Set PFC enable bit to asic only if the corresponding bit in user config is set
+    pfcMaskWdCfg = static_cast<uint8_t>(pfcMaskWdCfg | ((1 << getQueueId()) & pfcMaskUserCfg));
 
-    if (!gPortsOrch->setPortPfc(getPort(), pfcMask))
+    if (!gPortsOrch->setPortPfcStatus(getPort(), pfcMaskWdCfg))
     {
         SWSS_LOG_ERROR("Failed to set PFC mask on port 0x%" PRIx64, getPort());
     }

orchagent/pfcwdorch.cpp

@@ -324,7 +324,7 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::setBigRedSwitchMode(const string
 
     if (value == "enable")
     {
-        // When BIG_RED_SWITCH mode is enabled, pfcwd is automatically disabled
+        // When BIG_RED_SWITCH mode is enabled, pfcwd state machine is automatically disabled
         enableBigRedSwitchMode();
     }
     else if (value == "disable")
@@ -362,7 +362,8 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::disableBigRedSwitchMode()
         }
 
         auto queueId = entry.first;
-        string countersKey = this->getCountersTable()->getTableName() + this->getCountersTable()->getTableNameSeparator() + sai_serialize_object_id(queueId);
+        string countersKey = this->getCountersTable()->getTableName() + this->getCountersTable()->getTableNameSeparator()
+            + sai_serialize_object_id(queueId);
         this->getCountersDb()->hdel(countersKey, "BIG_RED_SWITCH_MODE");
     }
 
@@ -381,15 +382,16 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::enableBigRedSwitchMode()
     for (auto &it: allPorts)
     {
         Port port = it.second;
-        uint8_t pfcMask = 0;
+        uint8_t pfcMaskWdCfg = 0;
+        uint8_t dummy = 0;
 
         if (port.m_type != Port::PHY)
         {
             SWSS_LOG_INFO("Skip non-phy port %s", port.m_alias.c_str());
             continue;
         }
 
-        if (!gPortsOrch->getPortPfc(port.m_port_id, &pfcMask))
+        if (!gPortsOrch->getPortPfc(port.m_port_id, pfcMaskWdCfg, dummy))
         {
             SWSS_LOG_ERROR("Failed to get PFC mask on port %s", port.m_alias.c_str());
             return;
@@ -398,7 +400,11 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::enableBigRedSwitchMode()
         for (uint8_t i = 0; i < PFC_WD_TC_MAX; i++)
         {
             sai_object_id_t queueId = port.m_queue_ids[i];
-            if ((pfcMask & (1 << i)) == 0 && m_entryMap.find(queueId) == m_entryMap.end())
+            // PFC enable bit not set can be the case that the corresponding TC
+            // is lossless, and is currently in PFC storm, with PFC action in act.
+            // We pick up such a case to enable big red switch mode by checking if a corresponding
+            // entry exists in m_entryMap
+            if ((pfcMaskWdCfg & (1 << i)) == 0 && m_entryMap.find(queueId) == m_entryMap.end())
             {
                 continue;
             }
@@ -418,30 +424,37 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::enableBigRedSwitchMode()
         {
             entry.second.handler->commitCounters();
             entry.second.handler = nullptr;
+            // Remove storm status in APPL_DB for warm-reboot purpose
+            string key = m_applTable->getTableName() + m_applTable->getTableNameSeparator() + entry.second.portAlias;
+            m_applDb->hdel(key, to_string(entry.second.index));
         }
     }
 
-    // Create pfcwdaction handler on all the ports.
+    // Create pfcwdaction handler on all ports.
     for (auto & it: allPorts)
     {
         Port port = it.second;
-        uint8_t pfcMask = 0;
+        uint8_t pfcMaskWdCfg = 0;
+        uint8_t pfcMaskUserCfg = 0;
 
         if (port.m_type != Port::PHY)
         {
             SWSS_LOG_INFO("Skip non-phy port %s", port.m_alias.c_str());
             continue;
         }
 
-        if (!gPortsOrch->getPortPfc(port.m_port_id, &pfcMask))
+        if (!gPortsOrch->getPortPfc(port.m_port_id, pfcMaskWdCfg, pfcMaskUserCfg))
         {
             SWSS_LOG_ERROR("Failed to get PFC mask on port %s", port.m_alias.c_str());
             return;
         }
+        // By removing action handler, we expect PFC bit mask status in asic (pfcwd config) to
+        // be the same as user config
+        assert(pfcMaskWdCfg == pfcMaskUserCfg);
 
         for (uint8_t i = 0; i < PFC_WD_TC_MAX; i++)
         {
-            if ((pfcMask & (1 << i)) == 0)
+            if ((pfcMaskUserCfg & (1 << i)) == 0)
             {
                 continue;
             }
@@ -477,9 +490,10 @@ bool PfcWdSwOrch<DropHandler, ForwardHandler>::registerInWdDb(const Port& port,
 {
     SWSS_LOG_ENTER();
 
-    uint8_t pfcMask = 0;
+    uint8_t dummy = 0;
+    uint8_t pfcMaskUserCfg = 0;
 
-    if (!gPortsOrch->getPortPfc(port.m_port_id, &pfcMask))
+    if (!gPortsOrch->getPortPfc(port.m_port_id, dummy, pfcMaskUserCfg))
     {
         SWSS_LOG_ERROR("Failed to get PFC mask on port %s", port.m_alias.c_str());
         return false;
@@ -488,11 +502,12 @@ bool PfcWdSwOrch<DropHandler, ForwardHandler>::registerInWdDb(const Port& port,
     set<uint8_t> losslessTc;
     for (uint8_t i = 0; i < PFC_WD_TC_MAX; i++)
     {
-        if ((pfcMask & (1 << i)) == 0)
+        if ((pfcMaskUserCfg & (1 << i)) == 0)
         {
             continue;
         }
 
+        SWSS_LOG_NOTICE("Lossless TC %u found on port %s", i, port.m_alias.c_str());
         losslessTc.insert(i);
     }
     if (losslessTc.empty())
@@ -625,12 +640,16 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::unregisterFromWdDb(const Port& po
         if (entry != m_entryMap.end() && entry->second.handler != nullptr)
         {
             entry->second.handler->commitCounters();
+            // Remove storm status in APPL_DB for warm-reboot purpose
+            string key = m_applTable->getTableName() + m_applTable->getTableNameSeparator() + entry->second.portAlias;
+            m_applDb->hdel(key, to_string(entry->second.index));
         }
 
         m_entryMap.erase(queueId);
 
         // Clean up
-        string countersKey = this->getCountersTable()->getTableName() + this->getCountersTable()->getTableNameSeparator() + sai_serialize_object_id(queueId);
+        string countersKey = this->getCountersTable()->getTableName() + this->getCountersTable()->getTableNameSeparator()
+            + sai_serialize_object_id(queueId);
         this->getCountersDb()->hdel(countersKey, {"PFC_WD_DETECTION_TIME", "PFC_WD_RESTORATION_TIME", "PFC_WD_ACTION", "PFC_WD_STATUS"});
     }
 
@@ -889,7 +908,6 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::doTask(SelectableTimer &timer)
             handlerPair.second.handler->commitCounters(true);
         }
     }
-
 }
 
 template <typename DropHandler, typename ForwardHandler>

orchagent/port.h

@@ -126,7 +126,8 @@ class Port
     std::vector<sai_object_id_t> m_queue_ids;
     std::vector<sai_object_id_t> m_priority_group_ids;
     sai_port_priority_flow_control_mode_t m_pfc_asym = SAI_PORT_PRIORITY_FLOW_CONTROL_MODE_COMBINED;
-    uint8_t   m_pfc_bitmask = 0;
+    uint8_t   m_pfc_bitmask_usercfg = 0;    // bitmask from user config
+    uint8_t   m_pfc_bitmask_wdcfg = 0;      // bitmask of pfcwd config / status in asic
     uint32_t  m_nat_zone_id = 0;
     uint32_t  m_vnid = VNID_NONE;
     uint32_t  m_fdb_count = 0;