Add warm boot support with removed/created port (sonic-net#515)

* Add warm boot support with removed/created port

* Fix aspell

* Update sai_warmboot.bin location to local folder

Author: kcudnik

syncd/syncd.cpp

@@ -1257,6 +1257,7 @@ void get_port_related_objects(
 
 void post_port_remove(
         _In_ std::shared_ptr<SaiSwitch> sw,
+        _In_ sai_object_id_t port_rid,
         _In_ const std::vector<sai_object_id_t>& relatedRids)
 {
     SWSS_LOG_ENTER();
@@ -1307,16 +1308,16 @@ void post_port_remove(
 
         sai_object_type_t ot = redis_sai_object_type_query(vid);
 
-        std::string key = sai_serialize_object_type(ot) + ":" + str_vid;
+        std::string key = ASIC_STATE_TABLE + std::string(":") + sai_serialize_object_type(ot) + ":" + str_vid;
 
         SWSS_LOG_INFO("removing ASIC DB key: %s", key.c_str());
 
         g_redisClient->del(key);
     }
 
-    SWSS_LOG_NOTICE("post port remove actions succeeded");
+    sw->onPostPortRemove(port_rid);
 
-    // TODO lane map must be updated (for warm boot)
+    SWSS_LOG_NOTICE("post port remove actions succeeded");
 }
 
 void post_port_create(
@@ -1328,7 +1329,7 @@ void post_port_create(
 
     sw->onPostPortCreate(port_rid, port_vid);
 
-    // TODO lane map must be updated (for warm boot)
+    SWSS_LOG_NOTICE("post port create actions succeeded");
 }
 
 sai_status_t handle_generic(
@@ -1560,7 +1561,7 @@ sai_status_t handle_generic(
 
                         if (object_type == SAI_OBJECT_TYPE_PORT)
                         {
-                            post_port_remove(switches.at(switch_vid), related);
+                            post_port_remove(switches.at(switch_vid), rid, related);
                         }
                     }
                 }
@@ -2418,6 +2419,16 @@ sai_status_t processEventInInitViewMode(
     {
         case SAI_COMMON_API_CREATE:
 
+            if (object_type == SAI_OBJECT_TYPE_PORT)
+            {
+                // reason for this is that if user will create port,
+                // new port is not actually created so when for example
+                // querying new queues for new created port, there are
+                // not there, since no actual port create was issued on
+                // the ASIC
+                SWSS_LOG_THROW("port object can't be created in init view mode");
+            }
+
             if (info->isnonobjectid)
             {
                 /*
@@ -2450,6 +2461,17 @@ sai_status_t processEventInInitViewMode(
 
         case SAI_COMMON_API_REMOVE:
 
+            if (object_type == SAI_OBJECT_TYPE_PORT)
+            {
+                // reason for this is that if user will remove port, actual
+                // resources for it wont be release, lanes would be still
+                // occupied and there is extra logic required in post port
+                // remove which clears OIDs (ipgs,queues,SGs) from redis db
+                // that are automatically removed by vendor SAI, and comparison
+                // logic don't support that
+                SWSS_LOG_THROW("port object can't be removed in init view mode");
+            }
+
             if (object_type == SAI_OBJECT_TYPE_SWITCH)
             {
                 /*

syncd/syncd_applyview.cpp

@@ -6985,6 +6985,7 @@ void populateExistingObjects(
     auto sw = switches.begin()->second;
 
     auto coldBootDiscoveredVids = sw->getColdBootDiscoveredVids();
+    auto warmBootDiscoveredVids = sw->getWarmBootDiscoveredVids();
 
     /*
      * If some objects that are existing objects on switch are not present in
@@ -7058,9 +7059,47 @@ void populateExistingObjects(
          * NOTE: If we are here, then this RID exists only in current view, and
          * if this object contains any OID attributes, discovery logic queried
          * them so they are also existing in current view.
+         *
+         * Also in warm boot, when user removed port, and then created some new
+         * ports, new QUEUEs, IPGs and SGs will be created automatically by
+         * SAI.  Those new created objects mot likely will have different RID
+         * values then previous instances for given port. Those values should
+         * also be copied to temporary view, since they will not exist on cold
+         * boot discovered VIDs. If not, then comparison logic will try to remove
+         * them which is not what we want.
+         *
+         * This is tricky scenario, and there could be some issues also when
+         * other object types would be created by user.
          */
 
-        if (coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
+        bool performColdCheck = true;
+
+        if (warmBootDiscoveredVids.find(vid) != warmBootDiscoveredVids.end())
+        {
+            sai_object_type_t ot = redis_sai_object_type_query(vid);
+
+            switch (ot)
+            {
+                case SAI_OBJECT_TYPE_QUEUE:
+                case SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:
+                case SAI_OBJECT_TYPE_SCHEDULER_GROUP:
+
+                    // TODO this case may require adjustment, if user will do a
+                    // warm boot then remove/add some ports and make another
+                    // warm boot, it may happen that current logic will be
+                    // confused which of those objects are from previous warm
+                    // boot or second one, need better way to mark changes to
+                    // those objects in redis DB between warm boots
+
+                    performColdCheck = false;
+
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        if (performColdCheck && coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
         {
             SWSS_LOG_INFO("object is not on default existing list: %s RID %s VID %s",
                     sai_serialize_object_type(sai_object_type_query(rid)).c_str(),

syncd/syncd_hard_reinit.cpp

@@ -1349,7 +1349,7 @@ void performWarmRestart()
      * Perform all get operations on existing switch.
      */
 
-    auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid);
+    auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid, true);
 
     g_switch_rid = switch_rid;
     g_switch_vid = switch_vid;

syncd/syncd_saiswitch.cpp

@@ -7,6 +7,8 @@
 #include <unordered_map>
 #include <set>
 
+const int maxLanesPerPort = 8;
+
 /*
  * NOTE: all those methods could be implemented inside SaiSwitch class so then
  * we could skip using switch_id in params and even they could be public then.
@@ -148,18 +150,16 @@ std::unordered_map<sai_uint32_t, sai_object_id_t> SaiSwitch::saiGetHardwareLaneM
      * addressed in future.
      */
 
-    const int lanesPerPort = 8;
-
     for (const auto &port_rid : portList)
     {
-        sai_uint32_t lanes[lanesPerPort];
+        sai_uint32_t lanes[maxLanesPerPort];
 
         memset(lanes, 0, sizeof(lanes));
 
         sai_attribute_t attr;
 
         attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
-        attr.value.u32list.count = lanesPerPort;
+        attr.value.u32list.count = maxLanesPerPort;
         attr.value.u32list.list = lanes;
 
         sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);
@@ -1059,6 +1059,13 @@ std::set<sai_object_id_t> SaiSwitch::getColdBootDiscoveredVids() const
     return discoveredVids;
 }
 
+std::set<sai_object_id_t> SaiSwitch::getWarmBootDiscoveredVids() const
+{
+    SWSS_LOG_ENTER();
+
+    return m_warmBootDiscoveredVids;
+}
+
 void SaiSwitch::redisSaveColdBootDiscoveredVids() const
 {
     SWSS_LOG_ENTER();
@@ -1224,6 +1231,21 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(
     return ita->second;
 }
 
+void SaiSwitch::helperPopulateWarmBootVids()
+{
+    SWSS_LOG_ENTER();
+
+    if (!m_warmBoot)
+        return;
+
+    for (sai_object_id_t rid: m_discovered_rids)
+    {
+        sai_object_id_t vid = translate_rid_to_vid(rid, m_switch_vid);
+
+       m_warmBootDiscoveredVids.insert(vid);
+    }
+}
+
 /*
  * NOTE: If real ID will change during hard restarts, then we need to remap all
  * VID/RID, but we can only do that if we will save entire tree with all
@@ -1232,7 +1254,9 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(
 
 SaiSwitch::SaiSwitch(
         _In_ sai_object_id_t switch_vid,
-        _In_ sai_object_id_t switch_rid)
+        _In_ sai_object_id_t switch_rid,
+        _In_ bool warmBoot):
+    m_warmBoot(warmBoot)
 {
     SWSS_LOG_ENTER();
 
@@ -1263,9 +1287,68 @@ SaiSwitch::SaiSwitch(
 
     helperLoadColdVids();
 
+    helperPopulateWarmBootVids();
+
     saiGetMacAddress(m_default_mac_address);
 }
 
+std::vector<uint32_t> SaiSwitch::saiGetPortLanes(
+        _In_ sai_object_id_t port_rid)
+{
+    SWSS_LOG_ENTER();
+
+    std::vector<uint32_t> lanes;
+
+    lanes.resize(maxLanesPerPort);
+
+    sai_attribute_t attr;
+
+    attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
+    attr.value.u32list.count = maxLanesPerPort;
+    attr.value.u32list.list = lanes.data();
+
+    sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);
+
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_THROW("failed to get hardware lane list port RID %s: %s",
+                sai_serialize_object_id(port_rid).c_str(),
+                sai_serialize_status(status).c_str());
+    }
+
+    if (attr.value.u32list.count == 0)
+    {
+        SWSS_LOG_THROW("switch returned lane count ZERO for port RID %s",
+                sai_serialize_object_id(port_rid).c_str());
+    }
+
+    lanes.resize(attr.value.u32list.count);
+
+    return lanes;
+}
+
+void SaiSwitch::redisUpdatePortLaneMap(
+        _In_ sai_object_id_t port_rid)
+{
+    SWSS_LOG_ENTER();
+
+    auto lanes = saiGetPortLanes(port_rid);
+
+    for (uint32_t lane: lanes)
+    {
+        std::string strLane = sai_serialize_number(lane);
+        std::string strPortId = sai_serialize_object_id(port_rid);
+
+        auto key = getRedisLanesKey();
+
+        g_redisClient->hset(key, strLane, strPortId);
+    }
+
+    SWSS_LOG_NOTICE("added %zu lanes to redis lane map for port RID %s",
+            lanes.size(),
+            sai_serialize_object_id(port_rid).c_str());
+}
+
 void SaiSwitch::onPostPortCreate(
         _In_ sai_object_id_t port_rid,
         _In_ sai_object_id_t port_vid)
@@ -1282,7 +1365,8 @@ void SaiSwitch::onPostPortCreate(
 
     m_discovered_rids.insert(discovered.begin(), discovered.end());
 
-    SWSS_LOG_NOTICE("putting ALL new discovered objects to redis");
+    SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s",
+            sai_serialize_object_id(port_vid).c_str());
 
     for (sai_object_id_t rid: discovered)
     {
@@ -1296,5 +1380,48 @@ void SaiSwitch::onPostPortCreate(
 
         redisSetDummyAsicStateForRealObjectId(rid);
     }
+
+    redisUpdatePortLaneMap(port_rid);
 }
 
+void SaiSwitch::onPostPortRemove(
+        _In_ sai_object_id_t port_rid)
+{
+    SWSS_LOG_ENTER();
+
+    int removed = 0;
+
+    // key - lane number, value - port RID
+    auto map = redisGetLaneMap();
+
+    for (auto& kv: map)
+    {
+        if (kv.second == port_rid)
+        {
+            auto key = getRedisLanesKey();
+
+            std::string strLane = sai_serialize_number(kv.first);
+
+            g_redisClient->hdel(key, strLane);
+
+            removed++;
+        }
+    }
+
+    SWSS_LOG_NOTICE("removed %u lanes from redis lane map for port RID %s",
+            removed,
+            sai_serialize_object_id(port_rid).c_str());
+
+    if (removed == 0)
+    {
+        SWSS_LOG_THROW("NO LANES found in redis lane map for given port RID %s",
+            sai_serialize_object_id(port_rid).c_str());
+    }
+}
+
+bool SaiSwitch::isWarmBoot() const
+{
+    SWSS_LOG_ENTER();
+
+    return m_warmBoot;
+}