[services] optimize startup time on warm boot

files/build_templates/dhcp_relay.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=DHCP relay container
-Requires=updategraph.service swss.service teamd.service
-After=updategraph.service swss.service teamd.service
+Requires=updategraph.service swss.service syncd.service teamd.service
+After=updategraph.service swss.service syncd.service teamd.service
 Before=ntp-config.service
 
 [Service]

files/build_templates/lldp.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=LLDP container
 Requires=updategraph.service
-After=updategraph.service
+After=updategraph.service swss.service syncd.service
 Before=ntp-config.service
 
 [Service]

files/build_templates/radv.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=Router advertiser container
 Requires=updategraph.service
-After=updategraph.service swss.service
+After=updategraph.service swss.service syncd.service
 Before=ntp-config.service
 
 [Service]

files/build_templates/sflow.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=sFlow container
 Requires=swss.service
-After=swss.service
+After=swss.service syncd.service
 Before=ntp-config.service
 
 [Service]

files/build_templates/snmp.service.j2

@@ -2,13 +2,11 @@
 Description=SNMP container
 Requires=updategraph.service
 Requisite=swss.service
-After=updategraph.service swss.service
+After=updategraph.service swss.service syncd.service
 Before=ntp-config.service
 
 [Service]
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
-[Install]
-WantedBy=multi-user.target swss.service

files/build_templates/sonic_debian_extension.j2

@@ -317,7 +317,7 @@ EOF
 ## Bind docker path
 if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
     sudo mkdir -p $FILESYSTEM_ROOT/dockerfs
-    sudo mount --bind dockerfs $FILESYSTEM_ROOT/dockerfs 
+    sudo mount --bind dockerfs $FILESYSTEM_ROOT/dockerfs
 fi
 
 {% if installer_images.strip() -%}
@@ -334,7 +334,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS ta
 {% endif %}
 {% endfor %}
 if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
-    sudo umount $FILESYSTEM_ROOT/dockerfs 
+    sudo umount $FILESYSTEM_ROOT/dockerfs
     sudo rm -fr $FILESYSTEM_ROOT/dockerfs
     sudo kill -9 `sudo $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS_PID` || true
 else
@@ -362,6 +362,8 @@ sudo LANG=C cp $SCRIPTS_DIR/syncd.sh $FILESYSTEM_ROOT/usr/local/bin/syncd.sh
 # It implements delayed start of services
 sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer
+sudo cp $BUILD_TEMPLATES/telemetry.timer $FILESYSTEM_ROOT/etc/systemd/system/
+sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable telemetry.timer
 
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y

files/build_templates/telemetry.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=Telemetry container
 Requires=database.service
-After=database.service
+After=database.service swss.service syncd.service
 Before=ntp-config.service
 
 [Service]
@@ -10,5 +10,3 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
-[Install]
-WantedBy=multi-user.target

files/build_templates/telemetry.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Delays telemetry container until SONiC has started
+
+[Timer]
+OnBootSec=3min 30 sec
+Unit=telemetry.service
+
+[Install]
+WantedBy=timers.target

src/sonic-config-engine/lazy_re.py

@@ -0,0 +1,22 @@
+# monkey patch re.compile to improve import time of some packages
+
+import re
+
+_orig_re_compile = re.compile
+
+def __re_compile(*args, **kwargs):
+    class __LazyReCompile(object):
+        def __init__(self, *args, **kwargs):
+            self.args = args
+            self.kwargs = kwargs
+            self.pattern_obj = None
+
+        def __getattr__(self, name):
+            if self.pattern_obj is None:
+                self.pattern_obj = _orig_re_compile(*self.args, **self.kwargs)
+            return getattr(self.pattern_obj, name)
+    return  __LazyReCompile(*args, **kwargs)
+
+re.compile = __re_compile
+
+

src/sonic-config-engine/redis_bcc.py

@@ -0,0 +1,26 @@
+import jinja2
+
+class RedisBytecodeCache(jinja2.BytecodeCache):
+    """ A bytecode cache for jinja2 template that stores bytecode in Redis """
+
+    REDIS_HASH = 'JINJA2_CACHE'
+
+    def __init__(self, client):
+        self._client = client
+        try:
+            self._client.connect(self._client.STATE_DB, retry_on=False)
+        except Exception:
+            self._client = None
+
+    def load_bytecode(self, bucket):
+        if self._client is None:
+            return
+        code = self._client.get(self._client.STATE_DB, self.REDIS_HASH, bucket.key)
+        if code is not None:
+            bucket.bytecode_from_string(code)
+
+    def dump_bytecode(self, bucket):
+        if self._client is None:
+            return
+        self._client.set(self._client.STATE_DB, self.REDIS_HASH, bucket.key, bucket.bytecode_to_string())
+

src/sonic-config-engine/setup.py

@@ -16,7 +16,7 @@ def get_test_suite():
       author='Taoyu Li',
       author_email='[email protected]',
       url='https://github.com/Azure/sonic-buildimage',
-      py_modules=['portconfig', 'minigraph', 'openconfig_acl', 'sonic_device_util', 'config_samples'],
+      py_modules=['portconfig', 'minigraph', 'openconfig_acl', 'sonic_device_util', 'config_samples', 'redis_bcc', 'lazy_re'],
       scripts=['sonic-cfggen'],
       install_requires=['lxml', 'jinja2>=2.10', 'netaddr', 'ipaddr', 'pyyaml', 'pyangbind==0.6.0'],
       test_suite='setup.get_test_suite',

src/sonic-config-engine/sonic-cfggen

@@ -16,6 +16,16 @@ See usage string for detail description for arguments.
 """
 
 from __future__ import print_function
+
+# mokey patch re.compile to do lazy regular expression compilation.
+# This is done to improve import time of jinja2, yaml, natsort modules, because they
+# do many regexp compilation at import time, so it will speed up sonic-cfggen invocations
+# that do not require template generation or yaml loading. sonic-cfggen is used in so many places
+# during system boot up that importing jinja2, yaml, natsort every time
+# without lazy regular expression compilation affect boot up time.
+# FIXME: remove this once sonic-cfggen and templates dependencies are replaced with a faster approach
+import lazy_re
+
 import sys
 import os.path
 import argparse
@@ -33,7 +43,8 @@ from sonic_device_util import get_platform_info
 from sonic_device_util import get_system_mac
 from config_samples import generate_sample_config
 from config_samples import get_available_config
-from swsssdk import ConfigDBConnector
+from swsssdk import SonicV2Connector, ConfigDBConnector
+from redis_bcc import RedisBytecodeCache
 from collections import OrderedDict
 from natsort import natsorted
 
@@ -259,7 +270,8 @@ def main():
         paths  = ['/', '/usr/share/sonic/templates', os.path.dirname(template_file)]
         loader = jinja2.FileSystemLoader(paths)
 
-        env = jinja2.Environment(loader=loader, trim_blocks=True)
+        redis_bcc = RedisBytecodeCache(SonicV2Connector(host='127.0.0.1'))
+        env = jinja2.Environment(loader=loader, trim_blocks=True, bytecode_cache=redis_bcc)
         env.filters['sort_by_port_index'] = sort_by_port_index
         env.filters['ipv4'] = is_ipv4
         env.filters['ipv6'] = is_ipv6