Revert "[INDY-1225] Implemented compression, increased log rotation interval (hyperledger#607)"

This reverts commit 159a987.

Signed-off-by: Andrew Nikitin <[email protected]>

Author: Andrew Nikitin

plenum/config.py

@@ -149,11 +149,10 @@
 
 
 # Log configuration
-logRotationWhen = 'W6'
+logRotationWhen = 'D'
 logRotationInterval = 1
-logRotationBackupCount = 50
+logRotationBackupCount = 10
 logRotationMaxBytes = 100 * 1024 * 1024
-logRotationCompress = True
 logFormat = '{asctime:s} | {levelname:8s} | {filename:20s} ({lineno: >4}) | {funcName:s} | {message:s}'
 logFormatStyle = '{'
 logLevel = logging.NOTSET

plenum/test/test_log_rotation.py

@@ -2,50 +2,34 @@
 import logging
 import time
 import collections
-import gzip
-import pytest
 
 from stp_core.common.logging.TimeAndSizeRotatingFileHandler \
     import TimeAndSizeRotatingFileHandler
 from stp_core.common.log import Logger
 
 
-@pytest.fixture(params=[False, True], ids=["plain", "compressed"])
-def log_compression(request):
-    return request.param
-
-def test_default_log_rotation_config_is_correct(tdir_for_func):
-    logDirPath = tdir_for_func
-    logFile = os.path.join(logDirPath, "log")
-    logger = Logger()
-
-    # Assert this doesn't fail
-    logger.enableFileLogging(logFile)
-
-
-def test_time_log_rotation(tdir_for_func, log_compression):
+def test_time_log_rotation(tdir_for_func):
     logDirPath = tdir_for_func
     logFile = os.path.join(logDirPath, "log")
     logger = logging.getLogger('test_time_log_rotation-logger')
 
     logger.setLevel(logging.DEBUG)
-    handler = TimeAndSizeRotatingFileHandler(
-        logFile, interval=1, when='s', compress=log_compression)
+    handler = TimeAndSizeRotatingFileHandler(logFile, interval=1, when='s')
     logger.addHandler(handler)
     for i in range(3):
         time.sleep(1)
         logger.debug("line")
     assert len(os.listdir(logDirPath)) == 4  # initial + 3 new
 
 
-def test_size_log_rotation(tdir_for_func, log_compression):
+def test_size_log_rotation(tdir_for_func):
     logDirPath = tdir_for_func
     logFile = os.path.join(logDirPath, "log")
     logger = logging.getLogger('test_time_log_rotation-logger')
 
     logger.setLevel(logging.DEBUG)
     handler = TimeAndSizeRotatingFileHandler(
-        logFile, maxBytes=(4 + len(os.linesep)) * 4 + 1, compress=log_compression)
+        logFile, maxBytes=(4 + len(os.linesep)) * 4 + 1)
     logger.addHandler(handler)
     for i in range(20):
         logger.debug("line")
@@ -54,14 +38,14 @@ def test_size_log_rotation(tdir_for_func, log_compression):
     assert len(os.listdir(logDirPath)) == 5
 
 
-def test_time_and_size_log_rotation(tdir_for_func, log_compression):
+def test_time_and_size_log_rotation(tdir_for_func):
     logDirPath = tdir_for_func
     logFile = os.path.join(logDirPath, "log")
     logger = logging.getLogger('test_time_and_size_log_rotation-logger')
 
     logger.setLevel(logging.DEBUG)
     handler = TimeAndSizeRotatingFileHandler(
-        logFile, maxBytes=(4 + len(os.linesep)) * 4 + 1, interval=1, when="s", compress=log_compression)
+        logFile, maxBytes=(4 + len(os.linesep)) * 4 + 1, interval=1, when="s")
     logger.addHandler(handler)
 
     for i in range(20):
@@ -74,7 +58,7 @@ def test_time_and_size_log_rotation(tdir_for_func, log_compression):
     assert len(os.listdir(logDirPath)) == 8
 
 
-def test_time_and_size_log_rotation1(tdir_for_func, log_compression):
+def test_time_and_size_log_rotation1(tdir_for_func):
     log_dir_path = tdir_for_func
     logFile = os.path.join(log_dir_path, "log")
     logger = logging.getLogger('test_time_and_size_log_rotation-logger1')
@@ -90,7 +74,7 @@ def test_time_and_size_log_rotation1(tdir_for_func, log_compression):
     handler = TimeAndSizeRotatingFileHandler(
         logFile,
         maxBytes=(record_length + len(os.linesep)) * record_per_file + 1,
-        interval=1, when="h", backupCount=backup_count, utc=True, compress=log_compression)
+        interval=1, when="h", backupCount=backup_count, utc=True)
     logger.addHandler(handler)
 
     for i in range(1, record_count + 1):
@@ -107,7 +91,6 @@ def test_time_and_size_log_rotation1(tdir_for_func, log_compression):
     assert len(cir_buffer) == len(circ_buffer_set)
     assert len(os.listdir(log_dir_path)) == (backup_count + 1)
     for file_name in os.listdir(log_dir_path):
-        open_fn = gzip.open if file_name.endswith(".gz") else open
-        with open_fn(os.path.join(log_dir_path, file_name), "rt") as file:
+        with open(os.path.join(log_dir_path, file_name)) as file:
             for line in file.readlines():
                 assert line.strip() in circ_buffer_set

stp_core/common/log.py

@@ -97,8 +97,7 @@ def enableFileLogging(self, filename):
             interval=self._config.logRotationInterval,
             backupCount=self._config.logRotationBackupCount,
             utc=True,
-            maxBytes=self._config.logRotationMaxBytes,
-            compress=self._config.logRotationCompress)
+            maxBytes=self._config.logRotationMaxBytes)
         self._setHandler('file', new)
 
     def _setHandler(self, typ: str, new_handler):

stp_core/common/logging/TimeAndSizeRotatingFileHandler.py

@@ -1,5 +1,4 @@
 import os
-import gzip
 from logging.handlers import TimedRotatingFileHandler
 from logging.handlers import RotatingFileHandler
 
@@ -8,33 +7,21 @@ class TimeAndSizeRotatingFileHandler(TimedRotatingFileHandler, RotatingFileHandl
 
     def __init__(self, filename, when='h', interval=1, backupCount=0,
                  encoding=None, delay=False, utc=False, atTime=None,
-                 maxBytes=0, compress=False):
+                 maxBytes=0):
 
         TimedRotatingFileHandler.__init__(self, filename, when, interval,
                                           backupCount, encoding, delay,
                                           utc, atTime)
         self.maxBytes = maxBytes
-        self.compress = compress
 
     def shouldRollover(self, record):
         return bool(TimedRotatingFileHandler.shouldRollover(self, record)) or \
             bool(RotatingFileHandler.shouldRollover(self, record))
 
-    def rotate(self, source, dest):
-        source_gz = source.endswith(".gz")
-        dest_gz = dest.endswith(".gz")
-        if source_gz == dest_gz:
-            os.rename(source, dest)
-            return
-        # Assuming that not source_gz and dest_gz
-        with open(source, 'rb') as f_in, gzip.open(dest, 'wb') as f_out:
-            f_out.writelines(f_in)
-        os.remove(source)
-
     def rotation_filename(self, default_name: str):
-        compressed_name = self._compressed_filename(default_name)
-        if not os.path.exists(compressed_name):
-            return compressed_name
+
+        if not os.path.exists(default_name):
+            return default_name
 
         dir = os.path.dirname(default_name)
         defaultFileName = os.path.basename(default_name)
@@ -46,19 +33,13 @@ def rotation_filename(self, default_name: str):
                 index = self._file_index(fileName)
                 if index > maxIndex:
                     maxIndex = index
-        return self._compressed_filename("{}.{}".format(default_name, maxIndex + 1))
-
-    def _compressed_filename(self, file_name):
-        return "{}.gz".format(file_name) if self.compress else file_name
+        return "{}.{}".format(default_name, maxIndex + 1)
 
     @staticmethod
     def _file_index(file_name):
         split = file_name.split(".")
-        index = split[-1]
-        if index == "gz":
-            index = split[-2]
         try:
-            return int(index)
+            return int(split[-1])
         except ValueError:
             return 0
 
@@ -78,8 +59,6 @@ def getFilesToDelete(self):
         for fileName in fileNames:
             if fileName[:plen] == prefix:
                 suffix = fileName[plen:]
-                if suffix.endswith(".gz"):
-                    suffix = suffix[:-3]
                 if self.extMatch.match(suffix):
                     result.append(os.path.join(dirName, fileName))
         if len(result) <= self.backupCount:

stp_core/config.py

@@ -7,11 +7,10 @@
 baseDir = os.getcwd()
 
 # Log configuration
-logRotationWhen = 'W6'
+logRotationWhen = 'D'
 logRotationInterval = 1
-logRotationBackupCount = 50
+logRotationBackupCount = 10
 logRotationMaxBytes = 100 * 1024 * 1024
-logRotationCompress = True
 logFormat = '{asctime:s} | {levelname:8s} | {filename:20s} ({lineno:d}) | {funcName:s} | {message:s}'
 logFormatStyle = '{'