[sonic-package-manager] first phase implementation of sonic-package-manager

files/build_templates/sonic_debian_extension.j2

@@ -240,6 +240,16 @@ sudo rm -rf $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY3_WHEEL_NAME
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# Install SONiC Package Manager Python 3 package
+SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME=$(basename {{sonic_package_manager_py3_wheel_path}})
+sudo cp {{sonic_package_manager_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME
+
+# Install sonic-package-manager data files (and any dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-package-manager-data_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+
 # sonic-utilities-data installs bash-completion as a dependency. However, it is disabled by default
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/

rules/sonic-package-manager-data.dep

@@ -0,0 +1,9 @@
+SPATH       := $($(SONIC_PACKAGE_MANAGER_DATA)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-package-manager-data.mk rules/sonic-package-manager-data.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_PACKAGE_MANAGER_DATA)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_PACKAGE_MANAGER_DATA)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_PACKAGE_MANAGER_DATA)_DEP_FILES   := $(DEP_FILES)
+

rules/sonic-package-manager-data.mk

@@ -0,0 +1,6 @@
+# SONiC command line utilities data package
+
+SONIC_PACKAGE_MANAGER_DATA = sonic-package-manager-data_1.0-1_all.deb
+$(SONIC_PACKAGE_MANAGER_DATA)_SRC_PATH = $(SRC_PATH)/sonic-package-manager/sonic-package-manager-data
+SONIC_DPKG_DEBS += $(SONIC_PACKAGE_MANAGER_DATA)
+

rules/sonic-package-manager.dep

@@ -0,0 +1,11 @@
+SPATH       := $($(SONIC_PACKAGE_MANAGER_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-package-manager.mk rules/sonic-package-manager.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_PACKAGE_MANAGER_PY3)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_PACKAGE_MANAGER_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_PACKAGE_MANAGER_PY3)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_PACKAGE_MANAGER_PY3)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_PACKAGE_MANAGER_PY3)_SMDEP_PATHS := $(SPATH)
+

rules/sonic-package-manager.mk

@@ -0,0 +1,10 @@
+# sonic package manager python package
+
+SONIC_PACKAGE_MANAGER_PY3 = sonic_package_manager-1.0-py3-none-any.whl
+$(SONIC_PACKAGE_MANAGER_PY3)_SRC_PATH = $(SRC_PATH)/sonic-package-manager/
+$(SONIC_PACKAGE_MANAGER_PY3)_PYTHON_VERSION = 3
+$(SONIC_PACKAGE_MANAGER_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \
+                                        $(SWSSSDK_PY3) # Temporary dependency till issue with py-swsscommon is resolved
+$(SONIC_PACKAGE_MANAGER_PY3)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON)
+SONIC_PYTHON_WHEELS += $(SONIC_PACKAGE_MANAGER_PY3)
+

slave.mk

@@ -821,6 +821,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(PYTHON_SWSSCOMMON) \
                 $(PYTHON3_SWSSCOMMON) \
                 $(SONIC_UTILITIES_DATA) \
+                $(SONIC_PACKAGE_MANAGER_DATA) \
                 $(SONIC_HOST_SERVICES_DATA)) \
         $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
         $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
@@ -840,7 +841,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
-        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PACKAGE_MANAGER_PY3))
 	$(HEADER)
 	# Pass initramfs and linux kernel explicitly. They are used for all platforms
 	export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"
@@ -888,6 +890,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"
 	export sonic_utilities_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY3))"
+	export sonic_package_manager_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PACKAGE_MANAGER_PY3))"
 	export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))"
 
 	$(foreach docker, $($*_DOCKERS),\

src/sonic-package-manager/.gitignore

@@ -0,0 +1,142 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# IDEs
+.idea/
+.vscode/

src/sonic-package-manager/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --cov=sonic_package_manager --cov-report html --cov-report term --cov-report xml

src/sonic-package-manager/setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

src/sonic-package-manager/setup.py

@@ -0,0 +1,60 @@
+from setuptools import setup
+
+setup(
+    name='sonic-package-manager',
+    version='1.0',
+    description='SONiC Package Manager',
+    author='Stepan Blyshchak',
+    author_email='[email protected]',
+    url='https://github.com/Azure/sonic-buildimage',
+    packages=[
+        'sonic_package_manager',
+        'sonic_package_manager/service_creator'
+    ],
+    install_requires = [
+        'jinja2>=2.10',
+        'tabulate',
+        'poetry-semver',
+        'click',
+        'click_log',
+        'docker',
+        'docker-image-py',
+        'enlighten',
+        'natsort',
+        'prettyprinter',
+        'sonic-py-common',
+        'requests',
+        'www_authenticate',
+        'filelock',
+    ],
+    setup_requires=[
+        'pytest-runner',
+        'wheel',
+    ],
+    tests_require=[
+        'pytest',
+        'mock>=2.0.0',
+        'pytest-sugar',
+        'pytest-cov',
+        'pyfakefs',
+    ],
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    classifiers=[
+        'Intended Audience :: Developers',
+        'Natural Language :: English',
+        "Programming Language :: Python :: 2",
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+    ],
+    entry_points={
+        'console_scripts': [
+            'sonic-package-manager = sonic_package_manager.main:cli',
+            'spm = sonic_package_manager.main:cli',
+         ]
+    },
+    test_suite='setup.get_test_suite',
+)

src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/sonic-package-manager

@@ -0,0 +1,8 @@
+_sonic_package_manager_completion() {
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \
+                   COMP_CWORD=$COMP_CWORD \
+                   _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) )
+    return 0
+}
+
+complete -F _sonic_package_manager_completion -o default sonic-package-manager;

src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/spm

@@ -0,0 +1,8 @@
+_sonic_package_manager_completion() {
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \
+                   COMP_CWORD=$COMP_CWORD \
+                   _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) )
+    return 0
+}
+
+complete -F _sonic_package_manager_completion -o default spm;

src/sonic-package-manager/sonic-package-manager-data/debian/changelog

@@ -0,0 +1,5 @@
+sonic-package-manager-data (1.0-1) UNRELEASED; urgency=low
+
+  * Initial release
+
+ -- Stepan Blyshchak <[email protected]>  Thu, 1 Oct 2020 00:30:00 +0000

src/sonic-package-manager/sonic-package-manager-data/debian/compat

@@ -0,0 +1 @@
+9

src/sonic-package-manager/sonic-package-manager-data/debian/control

@@ -0,0 +1,11 @@
+Source: sonic-package-manager-data
+Maintainer: Stepan Blyshchak <[email protected]>
+Section: misc
+Priority: optional
+Standards-Version: 0.1
+Build-Depends: debhelper (>=9)
+
+Package: sonic-package-manager-data
+Architecture: all
+Depends: ${misc:Depends} bash-completion
+Description: Data files required for SONiC Package Manager

src/sonic-package-manager/sonic-package-manager-data/debian/install

@@ -0,0 +1,2 @@
+bash_completion.d/    /etc/
+templates/*.j2        /usr/share/sonic/templates/

src/sonic-package-manager/sonic-package-manager-data/debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+build:
+
+%:
+	dh $@

src/sonic-package-manager/sonic-package-manager-data/templates/dump.sh.j2

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#
+# =============== Managed by SONiC Package Manager. DO NOT EDIT! ===============
+#             auto-generated from {{ source }} by sonic-package-manager
+#
+
+service="{{ manifest.service.name }}"
+dump_command="{{ manifest.package['debug-dump'] }}"
+container_re="^${service}[0-9]*$"
+{% raw %}
+container_ids="$(docker ps -f name=${container_re} -f status=running --format {{.Names}})"
+{% endraw %}
+tmp_dir=$(mktemp -d)
+tmp_dump_dir="$tmp_dir/$service"
+tmp_archive=$(mktemp)
+
+mkdir -p "$tmp_dump_dir"
+
+for container_id in $container_ids; do
+    docker exec -t "${container_id}" ${dump_command} &> "${tmp_dump_dir}/${container_id}"
+done
+
+
+tar -C $(dirname $tmp_dump_dir) -cf $tmp_archive $service
+
+cat $tmp_archive
+rm $tmp_archive
+rm -rf $tmp_dir

src/sonic-package-manager/sonic-package-manager-data/templates/monit.conf.j2

@@ -0,0 +1,18 @@
+###############################################################################
+##
+## =============== Managed by SONiC Package Manager. DO NOT EDIT! ===============
+##             auto-generated from {{ source }} by sonic-package-manager
+##
+## Monit configuration for {{ feature }} service
+## process list:
+{%- for process in processes %}
+{%- if process.critical %}
+##  {{ process.name }}
+{%- endif %}
+{%- endfor %}
+###############################################################################
+{%- for process in processes %}
+check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}"
+    if status != 0 for 5 times within 5 cycles then alert
+
+{% endfor %}