diff --git a/setup.py b/setup.py index f074bc1c5dd..4b5290b7710 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ def read(*names, **kwargs): 'bz2file >= 0.98', # commoncode - 'backports.os == 0.1rc1', + 'backports.os == 0.1.1', 'future >= 0.16.0, < 0.17.0', 'text-unidecode >= 1.0, < 2.0', diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 89d85b1785a..ba1b6fd8e53 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -53,6 +53,11 @@ # Python 3 unicode = str +try: + from os import fsencode +except ImportError: + from backports.os import fsencode + """ Minimal wrapper for executing external commands in sub-processes. The approach @@ -344,7 +349,7 @@ def load_lib(libname, root_dir): if os.path.exists(so): if not isinstance(so, bytes): # ensure that the path is not Unicode... - so = so.encode(fileutils.FS_ENCODING) + so = fsencode(so) lib = ctypes.CDLL(so) if lib and lib._name: return lib diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 7f481fc539d..a0c236212e1 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -35,6 +35,13 @@ # Python 3 unicode = str +try: + from os import fsencode +except ImportError: + from backports.os import fsencode + from backports.os import fsdecode + + import codecs import errno import os @@ -45,7 +52,6 @@ import sys import tempfile -from backports import os as osb from commoncode import filetype from commoncode.filetype import is_rwx @@ -77,8 +83,6 @@ def logger_debug(*args): return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args)) -FS_ENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding() - # Paths can only be sanely handled as raw bytes on Linux PATH_TYPE = bytes if on_linux else unicode POSIX_PATH_SEP = b'/' if on_linux else '/' @@ -217,8 +221,8 @@ def path_to_unicode(path): """ if isinstance(path, unicode): return path - if TRACE: logger_debug('path_to_unicode:', osb.fsdecode(path)) - return osb.fsdecode(path) + if TRACE: logger_debug('path_to_unicode:', fsdecode(path)) + return fsdecode(path) def path_to_bytes(path): @@ -227,8 +231,8 @@ def path_to_bytes(path): """ if isinstance(path, bytes): return path - if TRACE: logger_debug('path_to_bytes:' , repr(osb.fsencode(path))) - return osb.fsencode(path) + if TRACE: logger_debug('path_to_bytes:' , repr(fsencode(path))) + return fsencode(path) def is_posixpath(location): diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index dfbb00cb61b..780fd74a29d 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -28,14 +28,14 @@ from __future__ import division from __future__ import unicode_literals -from unittest import TestCase as TestCaseClass - import filecmp +from functools import partial import os import shutil import stat import sys import tarfile +from unittest import TestCase as TestCaseClass import zipfile from commoncode import fileutils @@ -278,19 +278,27 @@ def extract_test_tar(self, test_path, verbatim=False): def extract_test_tar_raw(self, test_path, *args, **kwargs): return self.__extract(test_path, extract_tar_raw) + def extract_test_tar_unicode(self, test_path, *args, **kwargs): + return self.__extract(test_path, extract_tar_uni) -def extract_tar_raw(test_path, target_dir, *args, **kwargs): + +def _extract_tar_raw(test_path, target_dir, to_bytes, *args, **kwargs): """ Raw simplified extract for certain really weird paths and file names. """ - # use bytes for paths on ALL OSes (though this may fail on macOS) - target_dir = path_to_bytes(target_dir) - test_path = path_to_bytes(test_path) + if to_bytes: + # use bytes for paths on ALL OSes (though this may fail on macOS) + target_dir = path_to_bytes(target_dir) + test_path = path_to_bytes(test_path) tar = tarfile.open(test_path) tar.extractall(path=target_dir) tar.close() +extract_tar_raw = partial(_extract_tar_raw, to_bytes=True) + +extract_tar_uni = partial(_extract_tar_raw, to_bytes=False) + def extract_tar(location, target_dir, verbatim=False, *args, **kwargs): """ diff --git a/src/extractcode/libarchive2.py b/src/extractcode/libarchive2.py index 80601704b2e..40e0011e460 100644 --- a/src/extractcode/libarchive2.py +++ b/src/extractcode/libarchive2.py @@ -51,6 +51,13 @@ from extractcode import ExtractErrorPasswordProtected +# Python 2 and 3 support +try: + from os import fsencode +except ImportError: + from backports.os import fsencode + + logger = logging.getLogger(__name__) DEBUG = False # logging.basicConfig(level=logging.DEBUG) @@ -104,7 +111,7 @@ def load_lib(): if os.path.exists(libarchive): if not isinstance(libarchive, bytes): # ensure that the path is not Unicode... - libarchive = libarchive.encode(fileutils.FS_ENCODING) + libarchive = fsencode(libarchive) lib = ctypes.CDLL(libarchive) if lib and lib._name: return lib diff --git a/src/typecode/magic2.py b/src/typecode/magic2.py index e2a9b62d873..870a285b5d0 100644 --- a/src/typecode/magic2.py +++ b/src/typecode/magic2.py @@ -48,11 +48,17 @@ import os.path import ctypes -import sys from commoncode import system from commoncode import command +# Python 2 and 3 support +try: + from os import fsencode +except ImportError: + from backports.os import fsencode + + """ magic2 is minimal and specialized wrapper around a vendored libmagic file identification library. This is NOT thread-safe. It is based on python-magic @@ -204,7 +210,7 @@ def load_lib(): if os.path.exists(magic_so): if not isinstance(magic_so, bytes): # ensure that the path is not Unicode... - magic_so = magic_so.encode(sys.getfilesystemencoding() or sys.getdefaultencoding()) + magic_so = fsencode(magic_so) lib = ctypes.CDLL(magic_so) if lib and lib._name: return lib diff --git a/tests/scancode/data/unicode_fixtures.tar.gz b/tests/scancode/data/unicode_fixtures.tar.gz new file mode 100644 index 00000000000..6f395211122 Binary files /dev/null and b/tests/scancode/data/unicode_fixtures.tar.gz differ diff --git a/tests/scancode/data/unicode_fixtures/.hidden b/tests/scancode/data/unicode_fixtures/.hidden deleted file mode 100644 index b885243649b..00000000000 --- a/tests/scancode/data/unicode_fixtures/.hidden +++ /dev/null @@ -1 +0,0 @@ -I am hidden \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/empty.txt b/tests/scancode/data/unicode_fixtures/empty.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/scancode/data/unicode_fixtures/foo bar b/tests/scancode/data/unicode_fixtures/foo bar deleted file mode 100644 index 3f953866625..00000000000 --- a/tests/scancode/data/unicode_fixtures/foo bar +++ /dev/null @@ -1 +0,0 @@ -baz \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/nums b/tests/scancode/data/unicode_fixtures/nums deleted file mode 100644 index e2e107ac61a..00000000000 --- a/tests/scancode/data/unicode_fixtures/nums +++ /dev/null @@ -1 +0,0 @@ -123456789 \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/pets/names.txt b/tests/scancode/data/unicode_fixtures/pets/names.txt deleted file mode 100644 index 91407a3e048..00000000000 --- a/tests/scancode/data/unicode_fixtures/pets/names.txt +++ /dev/null @@ -1 +0,0 @@ -tobi,loki \ No newline at end of file diff --git "a/tests/scancode/data/unicode_fixtures/snow \342\230\203/.gitkeep" "b/tests/scancode/data/unicode_fixtures/snow \342\230\203/.gitkeep" deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/scancode/data/unicode_fixtures/todo.html b/tests/scancode/data/unicode_fixtures/todo.html deleted file mode 100644 index e7af6d7998d..00000000000 --- a/tests/scancode/data/unicode_fixtures/todo.html +++ /dev/null @@ -1 +0,0 @@ -
  • groceries
  • \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/todo.txt b/tests/scancode/data/unicode_fixtures/todo.txt deleted file mode 100644 index 8c3539d9461..00000000000 --- a/tests/scancode/data/unicode_fixtures/todo.txt +++ /dev/null @@ -1 +0,0 @@ -- groceries \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/users/index.html b/tests/scancode/data/unicode_fixtures/users/index.html deleted file mode 100644 index 00a2db41f75..00000000000 --- a/tests/scancode/data/unicode_fixtures/users/index.html +++ /dev/null @@ -1 +0,0 @@ -

    tobi, loki, jane

    \ No newline at end of file diff --git a/tests/scancode/data/unicode_fixtures/users/tobi.txt b/tests/scancode/data/unicode_fixtures/users/tobi.txt deleted file mode 100644 index 9d9529d47d7..00000000000 --- a/tests/scancode/data/unicode_fixtures/users/tobi.txt +++ /dev/null @@ -1 +0,0 @@ -ferret \ No newline at end of file diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index 61046ce121b..e99753eb4fc 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -36,6 +36,7 @@ # from click.testing import CliRunner from commoncode import fileutils +from commoncode.fileutils import path_to_bytes from commoncode.testcase import FileDrivenTesting from commoncode.system import on_linux from commoncode.system import on_mac @@ -47,7 +48,6 @@ from scancode.cli_test_utils import run_scan_plain from scancode import cli -from commoncode.fileutils import path_to_bytes test_env = FileDrivenTesting() @@ -206,7 +206,7 @@ def test_scan_mark_source_without_info(monkeypatch): result_file = test_env.get_temp_file('json') expected_file = test_env.get_test_loc('mark_source/without_info.expected.json') - result = run_scan_click(['--mark-source', test_dir, result_file], monkeypatch) + _result = run_scan_click(['--mark-source', test_dir, result_file], monkeypatch) check_json_scan(expected_file, result_file) def test_scan_mark_source_with_info(monkeypatch): @@ -214,7 +214,7 @@ def test_scan_mark_source_with_info(monkeypatch): result_file = test_env.get_temp_file('json') expected_file = test_env.get_test_loc('mark_source/with_info.expected.json') - result = run_scan_click(['--info', '--mark-source', test_dir, result_file], monkeypatch) + _result = run_scan_click(['--info', '--mark-source', test_dir, result_file], monkeypatch) check_json_scan(expected_file, result_file) def test_scan_only_findings(monkeypatch): @@ -222,7 +222,7 @@ def test_scan_only_findings(monkeypatch): result_file = test_env.get_temp_file('json') expected_file = test_env.get_test_loc('only_findings/expected.json') - result = run_scan_click(['--only-findings', test_dir, result_file], monkeypatch) + _result = run_scan_click(['--only-findings', test_dir, result_file], monkeypatch) check_json_scan(expected_file, result_file) @@ -451,11 +451,17 @@ def test_scan_does_not_fail_when_scanning_unicode_files_and_paths(): check_json_scan(test_env.get_test_loc(expected), result_file, strip_dates=True, regen=False) +@skipIf(on_windows, 'Python tar cannot extract these files on Windows') def test_scan_does_not_fail_when_scanning_unicode_test_files_from_express(): - test_dir = test_env.get_test_loc(u'unicode_fixtures') - if on_linux: - test_dir = path_to_bytes(test_dir) + # On Windows, Python tar cannot extract these files. Other + # extractors either fail or change the file name, making the test + # moot. Git cannot check these files. So for now it makes no sense + # to test this on Windows at all. Extractcode works fine, but does + # rename the problematic files. + + test_dir = test_env.extract_test_tar_raw(b'unicode_fixtures.tar.gz') + test_dir = path_to_bytes(test_dir) args = ['-n0', '--info', '--license', '--copyright', '--package', '--email', '--url', '--strip-root', diff --git a/thirdparty/prod/backports.os-0.1.1-py2.py3-none-any.whl b/thirdparty/prod/backports.os-0.1.1-py2.py3-none-any.whl new file mode 100644 index 00000000000..f04e5aa9567 Binary files /dev/null and b/thirdparty/prod/backports.os-0.1.1-py2.py3-none-any.whl differ diff --git a/thirdparty/prod/backports.os-0.1rc1-py2-none-any.whl b/thirdparty/prod/backports.os-0.1rc1-py2-none-any.whl deleted file mode 100644 index 9107c2bbe3f..00000000000 Binary files a/thirdparty/prod/backports.os-0.1rc1-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/prod/backports.os.ABOUT b/thirdparty/prod/backports.os.ABOUT index 84afe1e2691..9b30807e437 100644 --- a/thirdparty/prod/backports.os.ABOUT +++ b/thirdparty/prod/backports.os.ABOUT @@ -1,8 +1,8 @@ -about_resource: backports.os-0.1rc1-py2-none-any.whl -version: 0.1rc1 +about_resource: backports.os-0.1.1-py2.py3-none-any.whl +version: 0.1.1 name: backports.os home_url: https://github.com/pjdelport/backports.os -download_url: https://pypi.python.org/packages/82/1d/461604fd8b2c6f798a18fb9161019d35c005a489bdd91279dfcf9b65859e/backports.os-0.1rc1-py2-none-any.whl#md5=7677278e0dd1135b7c422495daa72573 +download_url: https://pypi.python.org/packages/5b/0b/5ba79ac5d09e0b38725498e0a212ace2d166c0b7a38a42045df824b68d59/backports.os-0.1.1-py2.py3-none-any.whl license_text: - backports.os.LICENSE - PSF.LICENSE diff --git a/thirdparty/prod/backports.os.LICENSE b/thirdparty/prod/backports.os.LICENSE index 41832c46962..5173da93702 100644 --- a/thirdparty/prod/backports.os.LICENSE +++ b/thirdparty/prod/backports.os.LICENSE @@ -1,4 +1,3 @@ - Copyright (c) Piƫt Delport, and Python Software Foundation Licensed under the Python license