[generate_dump] system dump improvements (sonic-net#503)

* [techsupport] add option to collect logs since given date

Allow user to dump logs newer some specific date to reduce
dump archive size

e.g:

admin@sonic:~$ show techsupport --since=yesterday

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] add specific error codes

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] exclude mellanox folders in /etc/

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] disable logrotate during log collection

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] add +w for procfs dump files

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] SINCE_DATE is epoch by default

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] change find_logs to find_files

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] add mstdump

Signed-off-by: Stepan Blyschak <[email protected]>

* [generate_dump] fix noop mode when generating sai/sdk/fw dump

Signed-off-by: Stepan Blyschak <[email protected]>

Author: stepanblyschak

scripts/generate_dump

@@ -6,6 +6,10 @@
 
 set -u
 
+ERROR_TAR_FAILED=5
+ERROR_PROCFS_SAVE_FAILED=6
+ERROR_INVALID_ARGUMENT=10
+
 TAR=tar
 MKDIR=mkdir
 RM=rm
@@ -14,10 +18,13 @@ GZIP=gzip
 CP=cp
 MV=mv
 GREP=grep
+TOUCH=touch
 V=
 NOOP=false
 DO_COMPRESS=true
 CMD_PREFIX=
+SINCE_DATE="@0" # default is set to January 1, 1970 at 00:00:00 GMT
+REFERENCE_FILE=/tmp/reference
 BASE=sonic_dump_`hostname`_`date +%Y%m%d_%H%M%S`
 DUMPDIR=/var/dump
 TARDIR=$DUMPDIR/$BASE
@@ -72,7 +79,7 @@ save_cmd() {
         fi
     fi
     ($TAR $V -rhf $TARFILE -C $DUMPDIR "$tarpath" \
-        || abort 5 "tar append operation failed. Aborting to prevent data loss.") \
+        || abort "${ERROR_TAR_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \
         && $RM $V -rf "$filepath"
 }
 
@@ -151,7 +158,7 @@ save_proc() {
     local procfiles="$@"
     $MKDIR $V -p $TARDIR/proc \
         && $CP $V -r $procfiles $TARDIR/proc \
-        && $TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+r $BASE/proc \
+        && $TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+rw $BASE/proc \
         && $RM $V -rf $TARDIR/proc
 }
 
@@ -212,10 +219,54 @@ save_file() {
         fi
     fi
     ($TAR $V -rhf $TARFILE -C $DUMPDIR "$tar_path" \
-        || abort 5 "tar append operation failed. Aborting to prevent data loss.") \
+        || abort "${ERROR_PROCFS_SAVE_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \
         && $RM $V -f "$gz_path"
 }
 
+###############################################################################
+# find_files routine
+# Globals:
+#  SINCE_DATE: list files only newer than given date
+#  REFERENCE_FILE: the file to be created as a reference to compare modification time
+# Arguments:
+#  directory: directory to search files in
+# Returns:
+#  None
+###############################################################################
+find_files() {
+    local -r directory=$1
+    $TOUCH --date="${SINCE_DATE}" "${REFERENCE_FILE}"
+    local -r find_command="find -L $directory -type f -newer ${REFERENCE_FILE}"
+
+    echo $($find_command)
+}
+
+###############################################################################
+# disable_logrotate routine
+# Globals:
+#  None
+# Arguments:
+#  None
+# Returns:
+#  None
+###############################################################################
+disable_logrotate() {
+    sed -i '/logrotate/s/^/#/g' /etc/cron.d/logrotate
+}
+
+###############################################################################
+# enable_logrotate routine
+# Globals:
+#  None
+# Arguments:
+#  None
+# Returns:
+#  None
+###############################################################################
+enable_logrotate() {
+    sed -i '/logrotate/s/^#*//g' /etc/cron.d/logrotate
+}
+
 ###############################################################################
 # Main generate_dump routine
 # Globals:
@@ -251,7 +302,7 @@ main() {
         /proc/softirqs /proc/stat /proc/swaps /proc/sysvipc /proc/timer_list \
         /proc/uptime /proc/version /proc/vmallocinfo /proc/vmstat \
         /proc/zoneinfo \
-        || abort 6 "Proc saving operation failed. Aborting for safety."
+        || abort "${ERROR_PROCFS_SAVE_FAILED}" "Proc saving operation failed. Aborting for safety."
 
     save_cmd "show version" "version"
     save_cmd "show platform summary" "platform.summary"
@@ -303,9 +354,16 @@ main() {
     local platform="$(/usr/local/bin/sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)"
     if [[ $platform == *"mlnx"* ]]; then
         local sai_dump_filename="/tmp/sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")"
-        docker exec -it syncd saisdkdump -f $sai_dump_filename
-        docker exec syncd tar Ccf $(dirname $sai_dump_filename) - $(basename $sai_dump_filename) | tar Cxf /tmp/ -
+        ${CMD_PREFIX}docker exec -it syncd saisdkdump -f $sai_dump_filename
+        ${CMD_PREFIX}docker exec syncd tar Ccf $(dirname $sai_dump_filename) - $(basename $sai_dump_filename) | tar Cxf /tmp/ -
         save_file $sai_dump_filename sai_sdk_dump true
+
+        local mst_dump_filename="/tmp/mstdump"
+        local max_dump_count="3"
+        for i in $(seq 1 $max_dump_count); do
+            ${CMD_PREFIX}/usr/bin/mstdump /dev/mst/mt*conf0 > "${mst_dump_filename}${i}"
+            save_file "${mst_dump_filename}${i}" mstdump true
+        done
     fi
 
     local asic="$(/usr/local/bin/sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)"
@@ -326,7 +384,7 @@ main() {
     $MKDIR $V -p $LOGDIR
     $LN $V -s /etc $TARDIR/etc
 
-    ($TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+r \
+    ($TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+rw \
         --exclude="etc/alternatives" \
         --exclude="*/etc/passwd*" \
         --exclude="*/etc/shadow*" \
@@ -335,12 +393,17 @@ main() {
         --exclude="*/etc/ssh*" \
         --exclude="*get_creds*" \
         --exclude="*snmpd.conf*" \
+        --exclude="/etc/mlnx" \
+        --exclude="/etc/mft" \
         $BASE/etc \
-        || abort 5 "Tar append operation failed. Aborting for safety.") \
+        || abort "${ERROR_TAR_FAILED}" "Tar append operation failed. Aborting for safety.") \
         && $RM $V -rf $TARDIR
 
+    disable_logrotate
+    trap enable_logrotate HUP INT QUIT TERM KILL ABRT ALRM
+
     # gzip up all log files individually before placing them in the incremental tarball
-    for file in $(find -L /var/log -type f); do
+    for file in $(find_files "/var/log/"); do
         # ignore the sparse file lastlog
         if [ "$file" = "/var/log/lastlog" ]; then
             continue
@@ -353,8 +416,10 @@ main() {
         fi
     done
 
+    enable_logrotate
+
     # archive core dump files
-    for file in $(find -L /var/core -type f); do
+    for file in $(find_files "/var/core/"); do
         # don't gzip already-gzipped log files :)
         if [ -z "${file##*.gz}" ]; then
             save_file $file core false
@@ -422,10 +487,15 @@ OPTIONS
         Noop mode. Don't actually create anything, just echo what would happen
     -z
         Don't compress the tar at the end.
+    -s DATE
+        Collect logs since DATE;
+        The argument is a mostly free format human readable string such as
+        "24 March", "yesterday", etc.
+
 EOF
 }
 
-while getopts ":xnvhz" opt; do
+while getopts ":xnvhzs:" opt; do
     case $opt in
         x)
             # enable bash debugging
@@ -450,11 +520,17 @@ while getopts ":xnvhz" opt; do
             CMD_PREFIX="echo "
             MV="echo mv"
             CP="echo cp"
+            TOUCH="echo touch"
             NOOP=true
             ;;
         z)
             DO_COMPRESS=false
             ;;
+        s)
+            SINCE_DATE="${OPTARG}"
+            # validate date expression
+            date --date="${SINCE_DATE}" &> /dev/null || abort "${ERROR_INVALID_ARGUMENT}" "Invalid date expression passed: '${SINCE_DATE}'"
+            ;;
         /?)
             echo "Invalid option: -$OPTARG" >&2
             exit 1

show/main.py

@@ -1282,10 +1282,13 @@ def users(verbose):
 #
 
 @cli.command()
+@click.option('--since', required=False, help="Collect logs and core files since given date")
 @click.option('--verbose', is_flag=True, help="Enable verbose output")
-def techsupport(verbose):
+def techsupport(since, verbose):
     """Gather information for troubleshooting"""
     cmd = "sudo generate_dump -v"
+    if since:
+        cmd += " -s {}".format(since)
     run_command(cmd, display_cmd=verbose)