T2-Route-Conv: Process Crash Optimization (sonic-net#603)

<!--
 Please make sure you've read and understood our contributing guidelines:
 https://github.com/Azure/SONiC/blob/gh-pages/CONTRIBUTING.md

 failure_prs.log skip_prs.log Make sure all your commits include a signature generated with `git commit -s` **

 If this is a bug fix, make sure your description includes "fixes #xxxx", or
 "closes #xxxx" or "resolves #xxxx"

 Please provide the following information:
-->

#### Why I did it
Fixes issue: sonic-net#21586

##### Work item tracking
- Microsoft ADO **31196012**:

#### How I did it
Run TSA-TSB service upon swss/swss0/swss1/.. startup. If the service is already running, reset the TSA-TSB timer.

#### How to verify it
Ran the T2 process crash sonic-mgmt snappi test to verify the convergence.
Before fix: ~10second
After Fix: <10ms

<!--
If PR needs to be backported, then the PR must be tested against the base branch and the earliest backport release branch and provide tested image version on these two branches. For example, if the PR is requested for master, 202211 and 202012, then the requester needs to provide test results on master and 202012.
-->

#### Which release branch to backport (provide reason below if selected)

<!--
- Note we only backport fixes to a release branch, *not* features!
- Please also provide a reason for the backporting below.
- e.g.
- [x] 202006
-->

- [ ] 201811
- [ ] 201911
- [ ] 202006
- [ ] 202012
- [ ] 202106
- [ ] 202111
- [ ] 202205
- [ ] 202211
- [ ] 202305

#### Tested branch (Please provide the tested image version)
SONiC.20240532.04
<!--
- Please provide tested image version
- e.g.
- [x] 20201231.100
-->

- [ ] <!-- image version 1 -->
- [ ] <!-- image version 2 -->

#### Description for the changelog
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->

<!--
 Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
-->

#### Link to config_db schema for YANG module changes
<!--
Provide a link to config_db schema for the table for which YANG model
is defined
Link should point to correct section on https://github.com/Azure/sonic-buildimage/blob/master/src/sonic-yang-models/doc/Configuration.md
-->

#### A picture of a cute animal (not mandatory but encouraged)

Author: mssonicbld

dockers/docker-fpm-frr/base_image_files/TSA

@@ -38,6 +38,11 @@ if [ -z "$STARTED_BY_TSA_TSB_SERVICE" ]; then
      [[ $(/bin/systemctl show $service --property SubState  --value) == "running" ]]; then
       echo "Stopping $service before configuring TSA"
       systemctl stop $service
+      if sonic-db-cli STATE_DB HDEL "ALL_SERVICE_STATUS|tsa_tsb_service" "running" >/dev/null; then
+        echo "Successfully removed TSA-TSB service flag."
+      else
+        echo "Failed to remove TSA-TSB service flag!" >&2
+      fi
   fi
 fi
 

dockers/docker-fpm-frr/base_image_files/TSB

@@ -37,6 +37,11 @@ if [ -z "$STARTED_BY_TSA_TSB_SERVICE" ]; then
      [[ $(/bin/systemctl show $service --property SubState  --value) == "running" ]]; then
       echo "Stopping $service before configuring TSB"
       systemctl stop $service
+      if sonic-db-cli STATE_DB HDEL "ALL_SERVICE_STATUS|tsa_tsb_service" "running" >/dev/null; then
+        echo "Successfully removed TSA-TSB service flag."
+      else
+        echo "Failed to remove TSA-TSB service flag!" >&2
+      fi
   fi
 fi
 

files/scripts/startup_tsa_tsb.py

@@ -69,19 +69,42 @@ def config_tsa():
     if tsa_ena == True:
         logger.log_info("Configuring TSA")
         subprocess.check_output(['TSA']).strip()
+        logger.log_info("Setting TSA-TSB service field in STATE_DB")
+        subprocess.check_output([
+            'sonic-db-cli', 'STATE_DB', 'HSET', 'ALL_SERVICE_STATUS|tsa_tsb_service', 'running', 'OK'
+        ]).strip()
     else:
-        if num_asics > 1:
-            logger.log_info("Either TSA is already configured or switch sub_role is not Frontend - not configuring TSA")
+        #check if tsa_tsb service is already running, restart the timer
+        try:
+            startup_tsa_tsb_service_status = subprocess.check_output([
+                'sonic-db-cli', 'STATE_DB', 'HGET', 'ALL_SERVICE_STATUS|tsa_tsb_service', 'running'
+            ]).strip().decode('utf-8')  # Convert bytes to string
+        except subprocess.CalledProcessError:
+            startup_tsa_tsb_service_status = None  # Default if the field is missing
+
+        if startup_tsa_tsb_service_status == 'OK':
+            logger.log_info("TSA-TSB service is already running, just restart the timer")
+            return True
         else:
-            logger.log_info("Either TSA is already configured - not configuring TSA")
+            if num_asics > 1:
+                logger.log_info("Either TSA is already configured or switch sub_role is not Frontend - not configuring TSA")
+            else:
+                logger.log_info("Either TSA is already configured - not configuring TSA")
     return tsa_ena
 
 def config_tsb():
     logger.log_info("Configuring TSB")
     subprocess.check_output(['TSB']).strip()
+
+    logger.log_info("Removing the TSA-TSB service field from STATE_DB")
+    subprocess.check_output([
+        'sonic-db-cli', 'STATE_DB', 'HDEL', 'ALL_SERVICE_STATUS|tsa_tsb_service', 'running'
+    ]).strip()
+
     tsb_issued = True
     return
 
+
 def start_tsb_timer(interval):
     global timer
     logger.log_info("Starting timer with interval {} seconds to configure TSB".format(interval))

files/scripts/swss.sh

@@ -8,7 +8,7 @@ DEBUGLOG="/tmp/swss-syncd-debug$DEV.log"
 LOCKFILE="/tmp/swss-syncd-lock$DEV"
 NAMESPACE_PREFIX="asic"
 ETC_SONIC_PATH="/etc/sonic/"
-
+TSA_TSB_SERVICE="startup_tsa_tsb.service"
 
 . /usr/local/bin/asic_status.sh
 
@@ -109,9 +109,9 @@ function clean_up_tables()
 
 # This function cleans up the chassis db table entries created ONLY by this asic
 # This is used to do the clean up operation when the line card / asic reboots
-# When the asic/lc is RE-booting, the chassis db server is supposed to be running 
-# in the supervisor.  So the clean up is done when only the chassis db connectable. 
-# Otherwise no need to do the clean up since both the supervisor and line card may be 
+# When the asic/lc is RE-booting, the chassis db server is supposed to be running
+# in the supervisor.  So the clean up is done when only the chassis db connectable.
+# Otherwise no need to do the clean up since both the supervisor and line card may be
 # rebooting (the whole chassis scenario)
 # The clean up operation is required to delete only those entries created by
 # the asic that is rebooted. Entries from the following tables are deleted in the order
@@ -212,7 +212,7 @@ function clean_up_chassis_db_tables()
     debug "Chassis db clean up for ${SERVICE}$DEV. Number of SYSTEM_LAG_MEMBER_TABLE entries deleted: $num_lag_mem"
 
     # Wait for some time before deleting system lag so that the all the memebers of the
-    # system lag will be cleared. 
+    # system lag will be cleared.
     # This delay is needed only if some system lag members were deleted
 
     if [[ $num_lag_mem > 0 ]]; then
@@ -258,6 +258,12 @@ start_peer_and_dependent_services() {
     check_warm_boot
 
     if [[ x"$WARM_BOOT" != x"true" ]]; then
+        SERVICES_CONF="/usr/share/sonic/device/$PLATFORM/services.conf"
+        if [[ -f $SERVICES_CONF ]] && grep -q "^startup_tsa_tsb.service$" $SERVICES_CONF; then
+            echo "${SERVICE}$DEV: starting TSA-TSB service"
+            /bin/systemctl restart $TSA_TSB_SERVICE
+        fi
+
         for peer in ${PEER}; do
             if [[ ! -z $DEV ]]; then
                 /bin/systemctl start ${peer}@$DEV