Skip to content

Commit 2f89949

Browse files
PterosaurPterosaur
authored
[dash-api]: Add dash-api and related protobuf library (#14515)
Why I did it For the DASH scenario, the APP_DB will be optimized by protobuf message for less memory consumption. How I did it Download the Debian package of protobuf 3.21.12 and create a corresponding rule for building it. Add a submodule of sonic-dash-api and generated its Debian package which includes C++ library and Python library How to verify it Check artifacts of Azp that the protobuf-related and dash-api deb packages should be generated. Signed-off-by: Ze Gan <[email protected]>
1 parent de65640 commit 2f89949

17 files changed

+262
-5
lines changed

‎.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# C/C++ with GCC
2+
# Build your C/C++ project with GCC using make.
3+
# Add steps that publish test results, save build artifacts, deploy, and more:
4+
# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
5+
pr: none
6+
7+
trigger:
8+
batch: true
9+
branches:
10+
include:
11+
- master
12+
13+
stages:
14+
- stage: Build
15+
16+
jobs:
17+
- job:
18+
displayName: "amd64/ubuntu-20.04"
19+
pool:
20+
vmImage: 'ubuntu-20.04'
21+
22+
steps:
23+
- checkout: self
24+
submodules: true
25+
- script: |
26+
sudo apt-get update
27+
sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
28+
cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
29+
debhelper-compat dh-elpa dh-sequence-python3 python3-all \
30+
libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
31+
ant default-jdk maven-repo-helper libguava-java
32+
wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
33+
sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
34+
mkdir -p /tmp/artifacts
35+
displayName: "Install dependencies"
36+
- script: |
37+
SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
38+
workingDirectory: src
39+
displayName: "Build protobuf"
40+
- script: |
41+
sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
42+
libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
43+
libprotobuf-lite32_3.21.12-3_amd64.deb
44+
workingDirectory: /tmp/artifacts
45+
displayName: "Install protobuf"
46+
- script: |
47+
dpkg-buildpackage -rfakeroot -b -us -uc
48+
workingDirectory: src/sonic-dash-api
49+
displayName: "Build sonic-dash-api"
50+
- script: |
51+
cp *.deb /tmp/artifacts
52+
workingDirectory: src
53+
- publish: /tmp/artifacts
54+
artifact: sonic-buildimage.amd64.ubuntu20_04
55+
displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,6 @@
115115
[submodule "src/dhcpmon"]
116116
path = src/dhcpmon
117117
url = https://github.com/sonic-net/sonic-dhcpmon.git
118+
[submodule "src/sonic-dash-api/sonic-dash-api"]
119+
path = src/sonic-dash-api/sonic-dash-api
120+
url = https://github.com/sonic-net/sonic-dash-api.git

‎rules/protobuf.dep

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
2+
SPATH := $($(PROTOBUF)_SRC_PATH)
3+
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/protobuf.mk rules/protobuf.dep
4+
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
5+
DEP_FILES += $(shell git ls-files $(SPATH))
6+
7+
$(PROTOBUF)_CACHE_MODE := GIT_CONTENT_SHA
8+
$(PROTOBUF)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
9+
$(PROTOBUF)_DEP_FILES := $(DEP_FILES)

‎rules/protobuf.mk

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# protobuf package
2+
3+
PROTOBUF_VERSION = 3.21.12
4+
PROTOBUF_VERSION_FULL = $(PROTOBUF_VERSION)-3
5+
6+
export PROTOBUF_VERSION
7+
export PROTOBUF_VERSION_FULL
8+
9+
PROTOBUF = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
10+
$(PROTOBUF)_SRC_PATH = $(SRC_PATH)/protobuf
11+
SONIC_MAKE_DEBS += $(PROTOBUF)
12+
13+
PROTOBUF_DEV = libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
14+
$(PROTOBUF_DEV)_DEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
15+
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_DEV)))
16+
17+
PROTOBUF_LITE = libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
18+
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_LITE)))
19+
20+
PROTOC = libprotoc_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
21+
$(PROTOC)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
22+
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC)))
23+
24+
PROTOC32 = libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
25+
$(PROTOC32)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE)
26+
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC32)))
27+
28+
PROTOBUF_COMPILER = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
29+
$(PROTOBUF_COMPILER)_DEPENDS = $(PROTOC32)
30+
$(PROTOBUF_COMPILER)_RDEPENDS = $(PROTOC)
31+
$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_COMPILER)))
32+
33+
PYTHON3_PROTOBUF = python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
34+
$(PYTHON3_PROTOBUF)_DEPENDS = $(PROTOBUF_DEV) $(PROTOBUF)
35+
$(PYTHON3_PROTOBUF)_RDEPENDS = $(PROTOBUF)
36+
$(eval $(call add_derived_package,$(PROTOBUF),$(PYTHON3_PROTOBUF)))

‎rules/sonic-dash-api.dep

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
SPATH := $($(LIB_SONIC_DASH_API)_SRC_PATH)
3+
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-dash-api.mk rules/sonic-dash-api.dep
4+
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
5+
SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
6+
7+
$(LIB_SONIC_DASH_API)_CACHE_MODE := GIT_CONTENT_SHA
8+
$(LIB_SONIC_DASH_API)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
9+
$(LIB_SONIC_DASH_API)_DEP_FILES := $(DEP_FILES)
10+
$(LIB_SONIC_DASH_API)_SMDEP_FILES := $(SMDEP_FILES)
11+
$(LIB_SONIC_DASH_API)_SMDEP_PATHS := $(SPATH)
12+

‎rules/sonic-dash-api.mk

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# libdashsai package
2+
3+
LIB_SONIC_DASH_API_VERSION = 1.0.0
4+
5+
LIB_SONIC_DASH_API = libdashapi_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
6+
$(LIB_SONIC_DASH_API)_SRC_PATH = $(SRC_PATH)/sonic-dash-api
7+
8+
$(LIB_SONIC_DASH_API)_DEPENDS += $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(PROTOBUF_COMPILER)
9+
$(LIB_SONIC_DASH_API)_RDEPENDS += $(LIBSWSSCOMMON) $(LIBNL3) $(LIBNL_GENL3) $(LIBNL_ROUTE3) $(PROTOBUF) $(PROTOBUF_LITE)
10+
11+
SONIC_DPKG_DEBS += $(LIB_SONIC_DASH_API)
12+
13+
LIB_SONIC_DASH_API_DBG = libdashapi-dbgsym_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb
14+
$(eval $(call add_derived_package,$(LIB_SONIC_DASH_API),$(LIB_SONIC_DASH_API_DBG)))
15+
16+
# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
17+
# are archived into debug one image to facilitate debugging.
18+
#
19+
DBG_SRC_ARCHIVE += sonic-dash-api
20+

‎rules/swss.mk

+4-2
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ SWSS = swss_1.0.0_$(CONFIGURED_ARCH).deb
44
$(SWSS)_SRC_PATH = $(SRC_PATH)/sonic-swss
55
$(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \
66
$(LIBTEAMDCTL) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \
7-
$(LIBSAIVS) $(LIBSAIVS_DEV)
7+
$(LIBSAIVS) $(LIBSAIVS_DEV) \
8+
$(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(LIB_SONIC_DASH_API)
89
$(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV)
910

1011
$(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) \
11-
$(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
12+
$(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) \
13+
$(PROTOBUF) $(PROTOBUF_LITE) $(PYTHON3_PROTOBUF) $(LIB_SONIC_DASH_API)
1214
SONIC_DPKG_DEBS += $(SWSS)
1315

1416
SWSS_DBG = swss-dbg_1.0.0_$(CONFIGURED_ARCH).deb

‎sonic-slave-bullseye/Dockerfile.j2

+7-3
Original file line numberDiff line numberDiff line change
@@ -344,8 +344,6 @@ RUN apt-get update && apt-get install -y \
344344
libnetfilter-conntrack-dev \
345345
libnftnl-dev \
346346
# For SAI3.7
347-
protobuf-compiler \
348-
libprotobuf-dev \
349347
xxd \
350348
# For DHCP Monitor tool
351349
libexplain-dev \
@@ -408,7 +406,13 @@ RUN apt-get update && apt-get install -y \
408406
pkg-config \
409407
# For audisp-tacplus
410408
libauparse-dev \
411-
auditd
409+
auditd \
410+
# For protobuf
411+
dh-elpa \
412+
xmlto \
413+
rake-compiler \
414+
default-jdk \
415+
libgoogle-gson-java
412416

413417
{%- if CROSS_BUILD_ENVIRON == "y" %}
414418
# Arm vs. amd64 versions conflict - remove amd64 packages

‎src/protobuf/Makefile

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
SHELL = /bin/bash
2+
.ONESHELL:
3+
.SHELLFLAGS += -e
4+
5+
MAIN_TARGET = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
6+
DERIVED_TARGETS = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
7+
libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
8+
libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
9+
libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \
10+
python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb
11+
12+
$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
13+
# Remove any stale files
14+
rm -rf protobuf-$(PROTOBUF_VERSION)
15+
16+
wget -O protobuf_$(PROTOBUF_VERSION).orig.tar.gz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION).orig.tar.gz
17+
wget -O protobuf_$(PROTOBUF_VERSION_FULL).dsc http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).dsc
18+
wget -O protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz
19+
dpkg-source -x protobuf_$(PROTOBUF_VERSION_FULL).dsc
20+
21+
pushd protobuf-$(PROTOBUF_VERSION)
22+
23+
ifeq ($(CROSS_BUILD_ENVIRON), y)
24+
dpkg-buildpackage -us -uc -b -a$(CONFIGURED_ARCH) -Pcross,nocheck -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
25+
else
26+
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
27+
endif
28+
29+
popd
30+
31+
mv $(DERIVED_TARGETS) $* $(DEST)/
32+
33+
$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

‎src/sonic-dash-api/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
build
2+
debian/sonic-dash-api

‎src/sonic-dash-api/Makefile

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
.ONESHELL:
2+
SHELL = /bin/bash
3+
.SHELLFLAGS += -e
4+
5+
RM := rm -rf
6+
CP := cp -rf
7+
MKDIR := mkdir
8+
MV := mv
9+
LIBDASHAPI := libdashapi.so
10+
BUILD_DIR := build
11+
PYPKG_DIR := pypkg
12+
DESTDIR :=
13+
DASH_API_PROTO_DIR := sonic-dash-api/proto
14+
INSTALLED_HEADER_DIR := $(DESTDIR)/usr/include/dash_api
15+
INSTALLED_LIB_DIR := $(DESTDIR)/usr/lib
16+
INSTALLED_PYTHON_DIR := $(DESTDIR)/usr/lib/python3/dist-packages/dash_api
17+
18+
all: compile_cpp_proto dashapi.so compile_py_proto
19+
20+
compile_cpp_proto:
21+
$(MKDIR) -p $(BUILD_DIR)
22+
protoc -I=$(DASH_API_PROTO_DIR) --cpp_out=$(BUILD_DIR) $(DASH_API_PROTO_DIR)/*.proto
23+
24+
dashapi.so: compile_cpp_proto
25+
g++ -std=c++14 -fPIC -shared -o $(BUILD_DIR)/$(LIBDASHAPI) $(wildcard $(BUILD_DIR)/*.pb.cc) -lprotobuf
26+
27+
compile_py_proto:
28+
protoc -I=$(DASH_API_PROTO_DIR) --python_out=$(PYPKG_DIR) $(DASH_API_PROTO_DIR)/*.proto
29+
30+
clean:
31+
$(RM) $(BUILD_DIR)
32+
$(RM) $(PYPKG_DIR)/*_pb2.py
33+
34+
install:
35+
$(MKDIR) -p $(INSTALLED_HEADER_DIR)
36+
$(CP) $(BUILD_DIR)/*.pb.h $(INSTALLED_HEADER_DIR)
37+
$(MKDIR) -p $(INSTALLED_LIB_DIR)
38+
$(CP) $(BUILD_DIR)/$(LIBDASHAPI) $(INSTALLED_LIB_DIR)
39+
$(MKDIR) -p $(INSTALLED_PYTHON_DIR)
40+
$(CP) $(PYPKG_DIR)/* $(INSTALLED_PYTHON_DIR)
41+
42+
uninstall:
43+
$(RM) $(INSTALLED_HEADER_DIR)
44+
$(RM) $(INSTALLED_LIB_DIR)/$(LIBDASHAPI)
45+
$(RM) $(INSTALLED_PYTHON_DIR)
46+
47+
.PHONY: uninstall clean

‎src/sonic-dash-api/debian/changelog

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
sonic (1.0.0) stable; urgency=medium
2+
3+
* Initial release.
4+
5+
-- Ze Gan <[email protected]> Wed, 14 Jun 2023 12:00:00 -0800
6+

‎src/sonic-dash-api/debian/compat

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
11

‎src/sonic-dash-api/debian/control

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Source: sonic
2+
Maintainer: Ze Gan <[email protected]>
3+
Section: net
4+
Priority: optional
5+
Build-Depends: dh-exec (>=0.3), debhelper (>= 12), autotools-dev,
6+
Standards-Version: 1.0.0
7+
8+
Package: libdashapi
9+
Architecture: any
10+
Build-Depends: protobuf-compiler (>=3.21.12), libprotobuf-dev (>=3.21.12)
11+
Depends: libprotobuf32 (>=3.21.12), libprotobuf-lite32 (>=3.21.12)
12+
Section: libs
13+
Description: DASH API definition for the SONiC project.

‎src/sonic-dash-api/debian/rules

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/make -f
2+
#export DH_VERBOSE = 1
3+
4+
%:
5+
dh $@
6+
7+
override_dh_auto_build:
8+
make all

‎src/sonic-dash-api/pypkg/__init__.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import sys
2+
import os
3+
4+
5+
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

‎src/sonic-dash-api/sonic-dash-api

Submodule sonic-dash-api added at 30415cd

0 commit comments

Comments
 (0)