Telemetry and mgmt-framework build fixes (#130)

1) Use buildimage-vs-image artifacts for sonic-mgmt-framework builds.
It requires buster packages.

2) Changed telemetry build script to install redis directly through apt.

3) Changed mgmt-framework to make it similar to telemetry build script's
layout -- install deps and compile sonic-mgmt-common by itself, without
depending on sonic-mgmt-common's build script.

Signed-off-by: Sachin Holla <[email protected]>

Author: sachinholla

jenkins/common/sonic-mgmt-common-build-pr/Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
                           userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common',
                                                refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]])
                 }
-                copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
+                copyArtifacts(projectName: 'vs/buildimage-vs-image', filter: '**/*.deb', target: 'buildimage', flatten: false)
             }
         }
 

jenkins/common/sonic-mgmt-framework-build-pr/Jenkinsfile

@@ -15,13 +15,12 @@ pipeline {
                           branches: [[name: 'refs/heads/master']],
                           userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]])
                 }
-                copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
+                copyArtifacts(projectName: 'vs/buildimage-vs-image', filter: '**/*.deb', target: 'buildimage', flatten: false)
             }
         }
 
         stage('Build') {
             steps {
-                sh './scripts/common/sonic-mgmt-common-build/build.sh'
                 sh './scripts/common/sonic-mgmt-framework-build/build.sh'
             }
         }

jenkins/common/sonic-mgmt-framework-build/Jenkinsfile

@@ -32,7 +32,6 @@ pipeline {
 
         stage('Build') {
             steps {
-                sh './scripts/common/sonic-mgmt-common-build/build.sh'
                 sh './scripts/common/sonic-mgmt-framework-build/build.sh'
             }
         }

jenkins/common/sonic-telemetry-build-pr/Jenkinsfile

@@ -10,6 +10,11 @@ pipeline {
                           userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-telemetry',
                                                refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]])
                 }
+                dir('sonic-mgmt-common') {
+                    checkout([$class: 'GitSCM',
+                          branches: [[name: 'refs/heads/master']],
+                          userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]])
+                }
                 copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
             }
         }

jenkins/common/sonic-telemetry-build/Jenkinsfile

@@ -21,6 +21,11 @@ pipeline {
                           branches: [[name: 'refs/heads/master']],
                           userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-telemetry']]])
                 }
+                dir('sonic-mgmt-common') {
+                    checkout([$class: 'GitSCM',
+                          branches: [[name: 'refs/heads/master']],
+                          userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]])
+                }
                 copyArtifacts(projectName: '../vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
             }
         }

scripts/common/sonic-mgmt-framework-build/build.sh

@@ -1,9 +1,43 @@
 #!/bin/bash -ex
 
 # Build script for sonic-mgmt-framework.
-# Assumes sonic-mgmt-common is already cloned & built.
-# Dependencies of sonic-mgmt-common would have been installed already.
-# Only sonic-mgmt-framework dependencies will have to be installed here.
+
+[[ ! -z ${DISTRO} ]] || DISTRO=buster
+
+# REDIS
+sudo apt-get install -y redis-server
+sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
+sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
+sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
+sudo service redis-server start
+
+# LIBYANG
+sudo dpkg -i buildimage/target/debs/${DISTRO}/libyang*.deb
+
+# Install from "requirement" files in sonic-mgmt-framework/tools/test directory.
+pushd sonic-mgmt-framework/tools/test
+
+[[ ! -f apt_requirements.txt ]] || \
+    sed 's/#.*//' apt_requirements.txt | xargs -r sudo apt-get install -y
+
+[[ ! -f python2_requirements.txt ]] || \
+    sudo pip install --no-cache-dir -r python2_requirements.txt
+
+[[ ! -f python3_requirements.txt ]] || \
+    sudo pip3 install --no-cache-dir -r python3_requirements.txt
+
+popd
+
+
+# Build sonic-mgmt-common
+
+pushd sonic-mgmt-common
+
+NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc
+
+popd
+
+# Build sonic-mgmt-framework
 
 pushd sonic-mgmt-framework
 
@@ -12,4 +46,4 @@ dpkg-buildpackage -rfakeroot -b -us -uc
 popd
 
 mkdir -p target
-cp sonic-mgmt-framework*.deb target/
+cp *.deb target/

scripts/common/sonic-mgmt-framework-build/test.sh

@@ -10,6 +10,7 @@ DEBDIR=$(realpath sonic-mgmt-common/debian/sonic-mgmt-common)
 pushd sonic-mgmt-framework/build/tests/rest
 
 export CVL_SCHEMA_PATH=${DEBDIR}/usr/sbin/schema
+export YANG_MODELS_PATH=${DEBDIR}/usr/models/yang
 
 ./server.test -test.v -logtostderr || STATUS=1
 

scripts/common/sonic-telemetry-build/build.sh

@@ -4,9 +4,7 @@
 sudo dpkg -i buildimage/target/debs/stretch/libhiredis*.deb
 
 # Install REDIS
-sudo apt-get install -y liblua5.1-0 lua-bitop lua-cjson
-sudo dpkg -i buildimage/target/debs/stretch/redis-tools_*.deb
-sudo dpkg -i buildimage/target/debs/stretch/redis-server_*.deb
+sudo apt-get install -y redis-server
 sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
 sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
 sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
@@ -15,14 +13,11 @@ sudo service redis-server start
 # Install libyang
 sudo dpkg -i buildimage/target/debs/stretch/libyang*.deb
 
-# Clone sonic-mgmt-common repository
-git clone https://github.com/Azure/sonic-mgmt-common.git
-
 # Build sonic-mgmt-common first
 
 pushd sonic-mgmt-common
 
-dpkg-buildpackage -rfakeroot -b -us -uc
+NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc
 
 popd