diff --git a/build_debian.sh b/build_debian.sh index 39b984edec4e..407cb8762521 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -348,9 +348,7 @@ sudo sed -i 's/^ListenAddress ::/#ListenAddress ::/' $FILESYSTEM_ROOT/etc/ssh/ss sudo sed -i 's/^#ListenAddress 0.0.0.0/ListenAddress 0.0.0.0/' $FILESYSTEM_ROOT/etc/ssh/sshd_config ## Config sysctl -sudo mkdir -p $FILESYSTEM_ROOT/var/core sudo augtool --autosave " -set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %t %p' set /files/etc/sysctl.conf/kernel.softlockup_panic 1 set /files/etc/sysctl.conf/kernel.panic 10 @@ -428,6 +426,8 @@ sudo cp files/dhcp/vrf $FILESYSTEM_ROOT/etc/dhcp/dhclient-exit-hooks.d/ if [ -f files/image_config/ntp/ntp ]; then sudo cp ./files/image_config/ntp/ntp $FILESYSTEM_ROOT/etc/init.d/ fi +## Configure application core dump handler +sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -y systemd-coredump ## Version file sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 167a392730e4..1bad7ac9118c 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -249,6 +249,9 @@ start() { docker create {{docker_image_run_opt}} \ {%- if install_debug_image == "y" %} -v /src:/src:ro -v /debug:/debug:rw \ + -v /var/log/journal:/var/log/journal:ro \ + -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ {%- endif %} {%- if '--log-driver=json-file' in docker_image_run_opt or '--log-driver' not in docker_image_run_opt %} --log-opt max-size=2M --log-opt max-file=5 \ diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index e5dd9d9e0912..a97f95a998b4 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -290,6 +290,24 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT/etc/systemd/s echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/ +# Allow systemd-coredump to perform cleanup of core files and not tmpfiles.d +sudo sed -i "/\/var\/lib\/systemd\/coredump/d" $FILESYSTEM_ROOT/usr/lib/tmpfiles.d/systemd.conf + +# Customize systemd-coredump configuration +sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/coredump.conf.d +sudo cp $IMAGE_CONFIGS/coredump/coredump.conf.d/00-sonic-coredump.conf $FILESYSTEM_ROOT/etc/systemd/coredump.conf.d +# Setup service to configure coredump service +sudo cp $IMAGE_CONFIGS/coredump/coredump-config.service $FILESYSTEM_ROOT/etc/systemd/system/ +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable coredump-config.service +sudo cp $IMAGE_CONFIGS/coredump/coredump-config.sh $FILESYSTEM_ROOT/usr/bin/ + +## Enable persistent journal to store coredump history +sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/journald.conf.d/ +sudo cp files/image_config/journald/journald.conf.d/00-sonic-journald.conf $FILESYSTEM_ROOT/etc/systemd/journald.conf.d/ + +## Shortcut to access core files +sudo ln -sf /var/lib/systemd/coredump $FILESYSTEM_ROOT/var/core + # Copy process/docker cpu/memory utilization data export daemon sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT/etc/systemd/system/ echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE diff --git a/files/image_config/coredump/coredump-config.service b/files/image_config/coredump/coredump-config.service new file mode 100644 index 000000000000..9aa5206ee333 --- /dev/null +++ b/files/image_config/coredump/coredump-config.service @@ -0,0 +1,11 @@ +[Unit] +Description=Update coredump configuration +Requires=updategraph.service +After=updategraph.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/coredump-config.sh + +[Install] +WantedBy=multi-user.target diff --git a/files/image_config/coredump/coredump-config.sh b/files/image_config/coredump/coredump-config.sh new file mode 100755 index 000000000000..ff08c4468577 --- /dev/null +++ b/files/image_config/coredump/coredump-config.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +DISABLE_COREDUMP_CONF="/etc/sysctl.d/50-disable-coredump.conf" + +if [ "$(redis-cli -n 4 HGET "COREDUMP|config" "enabled")" = "false" ] ; then + echo "kernel.core_pattern=" > ${DISABLE_COREDUMP_CONF} +else + rm -f ${DISABLE_COREDUMP_CONF} +fi + +# Read sysctl conf files again +systemctl restart systemd-sysctl + +exit 0 diff --git a/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf b/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf new file mode 100644 index 000000000000..5b08f7a0d185 --- /dev/null +++ b/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf @@ -0,0 +1,5 @@ +[Coredump] +Storage=external +Compress=yes +ProcessSizeMax=2G +ExternalSizeMax=2G diff --git a/files/image_config/journald/journald.conf.d/00-sonic-journald.conf b/files/image_config/journald/journald.conf.d/00-sonic-journald.conf new file mode 100644 index 000000000000..1055746ad8c4 --- /dev/null +++ b/files/image_config/journald/journald.conf.d/00-sonic-journald.conf @@ -0,0 +1,5 @@ +[Journal] +Storage=persistent +SystemMaxUse=256M +RuntimeMaxUse=356M +MaxLevelStore=crit diff --git a/rules/docker-base-stretch.mk b/rules/docker-base-stretch.mk index a54f4ec092aa..093579c5e953 100644 --- a/rules/docker-base-stretch.mk +++ b/rules/docker-base-stretch.mk @@ -11,7 +11,9 @@ VIM = vim OPENSSH = openssh-client SSHPASS = sshpass STRACE = strace -$(DOCKER_BASE_STRETCH)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE) +SYSTEMD_COREDUMP = systemd-coredump +$(DOCKER_BASE_STRETCH)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE) \ + $(SYSTEMD_COREDUMP) SONIC_DOCKER_IMAGES += $(DOCKER_BASE_STRETCH) SONIC_STRETCH_DOCKERS += $(DOCKER_BASE_STRETCH)