Add support for reconciliation after warm restart (sonic-net#76) (sonic-net#97)

Picking up commit below from master branch:
9044962 Jing Zhang Mon Jul 18 15:38:04 2022 -0700 Add support for reconciliation after warm restart (sonic-net#76)

Description of PR
Summary:
Fixes # (issue)

This PR is to add support for linkmgrd process reconciliation after warm restart.

sign-off: Jing Zhang [email protected]

What is the motivation for this PR?
One step of warm reboot procedure for dual ToR is to config the switch into manual mode. Before warm reboot finalizer executes config save, we want to config the switch back into auto mode, so config_db.json will be consistent before and after the reboot.

How did you do it?
When linkmgrd is initializing, get the systemwide warm reboot flag from WARM_RESTART_ENABLE_TABLE. If flag == true, start a reconciliation timer.
Maintenance a mux port count based on MUX_CABLE|PORTNAME count. When one port completes reconciliation, if warm restart flag == true, config it back into auto mode, reduce reconciliation port count by 1.
If reconciliation timer expires or port count == 0, set state to reconciled in WARM_RESTART_TABLE|linkmgrd.
How did you verify/test it?
Unit tests
Tested on dual ToR testbed. Ports were auto mode after warm restart completed. Entry WARM_RESTART_TABLE|linkmgrd was added as expected.

Author: zjswhhh

src/DbInterface.cpp

@@ -581,9 +581,58 @@ void DbInterface::getServerIpAddress(std::shared_ptr<swss::DBConnector> configDb
     std::vector<swss::KeyOpFieldsValuesTuple> entries;
 
     configDbMuxCableTable.getContent(entries);
+    mMuxManagerPtr->updateWarmRestartReconciliationCount(entries.size());
     processServerIpAddress(entries);
 }
 
+// ---> warmRestartReconciliation(const std::string &portName);
+//
+// port warm restart reconciliation procedure
+//
+void DbInterface::warmRestartReconciliation(const std::string &portName)
+{
+    MUXLOGDEBUG(portName);
+
+    if (isWarmStart()) {
+        setMuxMode(portName, "auto");
+        mMuxManagerPtr->updateWarmRestartReconciliationCount(-1);
+    }
+}
+
+//
+// ---> setMuxMode
+//
+// set config db mux mode
+//
+void DbInterface::setMuxMode(const std::string &portName, const std::string state)
+{
+    MUXLOGDEBUG(portName);
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &DbInterface::handleSetMuxMode,
+        this,
+        portName,
+        state
+    )));
+}
+
+//
+// ---> handleSetMuxmode
+//
+// handle set mux mode
+//
+void DbInterface::handleSetMuxMode(const std::string &portName, const std::string state)
+{
+    MUXLOGWARNING(boost::format("%s: configuring mux mode to %s after warm restart") % portName % state);
+
+    std::shared_ptr<swss::DBConnector> configDbPtr = std::make_shared<swss::DBConnector> ("CONFIG_DB", 0);
+    std::shared_ptr<swss::Table> configDbMuxCableTablePtr = std::make_shared<swss::Table> (
+        configDbPtr.get(), CFG_MUX_CABLE_TABLE_NAME
+    );
+    configDbMuxCableTablePtr->hset(portName, "state", state);
+}
+
 //
 // ---> processMuxPortConfigNotifiction(std::deque<swss::KeyOpFieldsValuesTuple> &entries);
 //

src/DbInterface.h

@@ -34,6 +34,7 @@
 #include "swss/dbconnector.h"
 #include "swss/producerstatetable.h"
 #include "swss/subscriberstatetable.h"
+#include "swss/warm_restart.h"
 
 #include "link_manager/LinkManagerStateMachine.h"
 #include "mux_state/MuxState.h"
@@ -245,6 +246,56 @@ class DbInterface
     */
     void stopSwssNotificationPoll() {mPollSwssNotifcation = false;};
 
+    /**
+     * @method setMuxMode 
+     * 
+     * @brief set config db mux mode 
+     * 
+     * @param portName (in) MUX port name 
+     * @param state (in) MUX mode state 
+     *  
+     * @return none
+     */
+    void setMuxMode(const std::string &portName, const std::string state);
+
+    /**
+     * @method warmRestartReconciliation
+     * 
+     * @brief port warm restart reconciliation procedure
+     * 
+     * @param portName(in) Mux port name
+     * 
+     * @return none
+     */
+    void warmRestartReconciliation(const std::string &portName);
+
+    /**
+     * @method isWarmStart
+     * 
+     * @brief is warm start or not
+     * 
+     * @return system flag for warm start context 
+     */
+    virtual bool isWarmStart(){return swss::WarmStart::isWarmStart();};
+
+    /**
+     * @method getWarmStartTimer
+     * 
+     * @brief get warm start time out in sec
+     * 
+     * @return timeout in sec 
+     */
+    virtual uint32_t getWarmStartTimer(){return swss::WarmStart::getWarmStartTimer("linkmgrd", "mux");};
+
+    /**
+     * @method setWarmStartStateReconciled
+     * 
+     * @brief set warm start state reconciled
+     * 
+     * @return none
+     */
+    virtual void setWarmStartStateReconciled(){swss::WarmStart::setWarmStartState("linkmgrd", swss::WarmStart::RECONCILED);};
+
 private:
     friend class test::MuxManagerTest;
 
@@ -347,6 +398,18 @@ class DbInterface
         const uint64_t expectedPacketCount
     );
 
+    /**
+     * @method handleSetMuxMode
+     * 
+     * @brief handle set mux mode 
+     * 
+     * @param portName (in) MUX port name
+     * @param state (in) MUX mode state
+     * 
+     * @return none
+     */
+    virtual void handleSetMuxMode(const std::string &portName, const std::string state);
+
     /**
     *@method processTorMacAddress
     *

src/LinkMgrdMain.cpp

@@ -26,6 +26,8 @@
 #include <boost/lexical_cast.hpp>
 #include <boost/program_options.hpp>
 
+#include "swss/warm_restart.h"
+
 #include "MuxManager.h"
 #include "MuxPort.h"
 #include "common/MuxConfig.h"
@@ -123,6 +125,13 @@ int main(int argc, const char* argv[])
         link_prober::IcmpPayload::generateGuid();
         link_manager::LinkManagerStateMachine::initializeTransitionFunctionTable();
 
+        // warm restart static
+        swss::WarmStart::initialize("linkmgrd", "mux");
+        swss::WarmStart::checkWarmStart("linkmgrd", "mux");
+        if (swss::WarmStart::isWarmStart()) {
+            swss::WarmStart::setWarmStartState("linkmgrd", swss::WarmStart::INITIALIZED);
+        }
+
         std::shared_ptr<mux::MuxManager> muxManagerPtr = std::make_shared<mux::MuxManager> ();
         muxManagerPtr->initialize(measureSwitchover, defaultRoute);
         muxManagerPtr->run();

src/MuxManager.cpp

@@ -43,7 +43,9 @@ MuxManager::MuxManager() :
     mMuxConfig(),
     mWork(mIoService),
     mSignalSet(boost::asio::signal_set(mIoService, SIGINT, SIGTERM)),
-    mDbInterfacePtr(std::make_shared<mux::DbInterface> (this, &mIoService))
+    mDbInterfacePtr(std::make_shared<mux::DbInterface> (this, &mIoService)),
+    mStrand(mIoService),
+    mReconciliationTimer(mIoService)
 {
     mSignalSet.add(SIGUSR1);
     mSignalSet.add(SIGUSR2);
@@ -70,6 +72,11 @@ void MuxManager::initialize(bool enable_feature_measurement, bool enable_feature
 
     mDbInterfacePtr->initialize();
 
+    if (mDbInterfacePtr->isWarmStart()) {
+        MUXLOGINFO("Detected warm restart context, starting reconciliation timer.");
+        startWarmRestartReconciliationTimer(mDbInterfacePtr->getWarmStartTimer());
+    }
+
     mMuxConfig.enableSwitchoverMeasurement(enable_feature_measurement);
     mMuxConfig.enableDefaultRouteFeature(enable_feature_default_route);
 }
@@ -363,4 +370,65 @@ void MuxManager::handleProcessTerminate()
     mDbInterfacePtr->getBarrier().wait();
 }
 
+// ---> updateWarmRestartReconciliationCount(int increment);
+//
+// update warm restart reconciliation count
+//
+void MuxManager::updateWarmRestartReconciliationCount(int increment)
+{
+    MUXLOGDEBUG(increment);
+
+    boost::asio::io_service &ioService = mStrand.context();
+
+    ioService.post(mStrand.wrap(boost::bind(
+        &MuxManager::handleUpdateReconciliationCount,
+        this,
+        increment
+    )));
+}
+
+// ---> handleUpdateReconciliationCount(int increment);
+//
+// handler of updating reconciliation port count
+// 
+void MuxManager::handleUpdateReconciliationCount(int increment)
+{
+    MUXLOGDEBUG(mPortReconciliationCount);
+
+    mPortReconciliationCount += increment;
+
+    if(mPortReconciliationCount == 0) {
+        mReconciliationTimer.cancel();
+    } 
+}
+
+// ---> startWarmRestartReconciliationTimer
+//
+// start warm restart reconciliation timer
+// 
+void MuxManager::startWarmRestartReconciliationTimer(uint32_t timeout)
+{
+    mReconciliationTimer.expires_from_now(boost::posix_time::seconds(
+        timeout == 0? mMuxConfig.getMuxReconciliationTimeout_sec():timeout
+    ));
+    mReconciliationTimer.async_wait(mStrand.wrap(boost::bind(
+        &MuxManager::handleWarmRestartReconciliationTimeout,
+        this,
+        boost::asio::placeholders::error
+    )));
+}
+
+// ---> handleWarmRestartReconciliationTimeout
+// 
+// handle warm restart reconciliationTimeout
+//
+void MuxManager::handleWarmRestartReconciliationTimeout(const boost::system::error_code errorCode)
+{
+    if (errorCode == boost::system::errc::success) {
+        MUXLOGWARNING("Reconciliation timed out after warm restart, set service to reconciled now.");
+    }
+
+    mDbInterfacePtr->setWarmStartStateReconciled();
+}
+
 } /* namespace mux */

src/MuxManager.h

@@ -353,6 +353,17 @@ class MuxManager
     */
     void addOrUpdateDefaultRouteState(bool is_v4, const std::string &routeState);
 
+    /**
+     * @method updateWarmRestartReconciliationCount
+     * 
+     * @brief update warm restart reconciliation count
+     * 
+     * @param increment
+     * 
+     * @return none
+     */
+    void updateWarmRestartReconciliationCount(int increment);
+
 private:
     /**
     *@method getMuxPortPtrOrThrow
@@ -397,6 +408,38 @@ class MuxManager
     */
     void setDbInterfacePtr(std::shared_ptr<mux::DbInterface> dbInterfacePtr) {mDbInterfacePtr = dbInterfacePtr;};
 
+private: 
+    /**
+     * @method startWarmRestartReconciliationTimer
+     * 
+     * @brief start warm restart reconciliation timer
+     * 
+     * @return none
+     */
+    void startWarmRestartReconciliationTimer(uint32_t timeout=0);
+
+    /**
+     * @method handleWarmRestartReconciliationTimeout
+     * 
+     * @brief handle warm restart reconciliationTimeout
+     *
+     * @param errorCode (in) Boost error code 
+     *  
+     * @return none
+     */
+    void handleWarmRestartReconciliationTimeout(const boost::system::error_code errorCode);
+
+    /**
+     * @method handleUpdateReconciliationCount
+     * 
+     * @brief handler of updating reconciliation port count 
+     * 
+     * @param increment
+     * 
+     * @return none
+     */
+    void handleUpdateReconciliationCount(int increment);
+
 private:
     common::MuxConfig mMuxConfig;
 
@@ -405,6 +448,10 @@ class MuxManager
     boost::thread_group mThreadGroup;
     boost::asio::signal_set mSignalSet;
 
+    boost::asio::io_service::strand mStrand;
+    boost::asio::deadline_timer mReconciliationTimer;
+    uint16_t mPortReconciliationCount = 0;
+
     std::shared_ptr<mux::DbInterface> mDbInterfacePtr;
 
     PortMap mPortMap;

src/MuxPort.cpp

@@ -288,4 +288,16 @@ void MuxPort::resetPckLossCount()
     )));
 }
 
+//
+// ---> warmRestartReconciliation();
+//
+// brief port warm restart reconciliation procedure
+//
+void MuxPort::warmRestartReconciliation()
+{
+    if (mMuxPortConfig.getMode() != common::MuxPortConfig::Mode::Auto) {
+        mDbInterfacePtr->warmRestartReconciliation(mMuxPortConfig.getPortName());
+    }
+}
+
 } /* namespace mux */

src/MuxPort.h

@@ -310,6 +310,15 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
     */
     void resetPckLossCount();
 
+    /**
+     * @method warmRestartReconciliation
+     * 
+     * @brief port warm restart reconciliation procedure
+     * 
+     * @return none
+     */
+    void warmRestartReconciliation();
+
 protected:
     friend class test::MuxManagerTest;
     friend class test::FakeMuxPort;

src/common/MuxConfig.h

@@ -316,6 +316,15 @@ class MuxConfig
      */
     inline bool getIfEnableDefaultRouteFeature() {return mEnableDefaultRouteFeature;};
 
+    /**
+     * @method getMuxReconciliationTimeout
+     * 
+     * @brief getter of mux reconciliation time out 
+     * 
+     * @return timeout in sec
+     */
+    inline uint32_t getMuxReconciliationTimeout_sec(){return mMuxReconciliationTimeout_sec;};
+
 private:
     uint8_t mNumberOfThreads = 5;
     uint32_t mTimeoutIpv4_msec = 100;
@@ -329,6 +338,8 @@ class MuxConfig
     bool mEnableSwitchoverMeasurement = false;
     uint32_t mDecreasedTimeoutIpv4_msec = 10;
 
+    uint32_t mMuxReconciliationTimeout_sec = 10;
+
     bool mEnableDefaultRouteFeature = false;
 
     std::array<uint8_t, ETHER_ADDR_LEN> mTorMacAddress;

src/link_manager/LinkManagerStateMachine.cpp

@@ -482,6 +482,8 @@ void LinkManagerStateMachine::activateStateMachine()
         mStartProbingFnPtr();
 
         updateMuxLinkmgrState();
+
+        mMuxPortPtr->warmRestartReconciliation();
     }
 }