pylint/2.16.2: tests failures, test_allow_reexport_package and test_writing_minimal_file

[sandrotosi]

Bug description

when running unittests, 2 tests fail

Command used

pytest

Pylint output

=================================================================================================================================== FAILURES ===================================================================================================================================
________________________________________________________________________________________________________________ TestImportsChecker.test_allow_reexport_package ________________________________________________________________________________________________________________

capsys = <_pytest.capture.CaptureFixture object at 0x7f02afeb9290>

    @staticmethod
    def test_allow_reexport_package(capsys: CaptureFixture[str]) -> None:
        """Test --allow-reexport-from-package option."""
    
        # Option disabled - useless-import-alias should always be emitted
        Run(
            [
                f"{os.path.join(REGR_DATA, 'allow_reexport')}",
                "--allow-reexport-from-package=no",
                "-sn",
            ],
            exit=False,
        )
        output, errors = capsys.readouterr()
>       assert len(output.split("\n")) == 5
E       AssertionError: assert 7 == 5
E        +  where 7 = len(['************* Module allow_reexport', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module docstring (missing-module-docstring)', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0414: Import alias does not rename ori
ginal package (useless-import-alias)', '************* Module allow_reexport.file', 'tests/regrtest_data/allow_reexport/file.py:1:0: C0114: Missing module docstring (missing-module-docstring)', 'tests/regrtest_data/allow_reexport/file.py:2:0: C0414: Import alias does not r
ename original package (useless-import-alias)', ...])
E        +    where ['************* Module allow_reexport', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module docstring (missing-module-docstring)', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0414: Import alias does not rename original 
package (useless-import-alias)', '************* Module allow_reexport.file', 'tests/regrtest_data/allow_reexport/file.py:1:0: C0114: Missing module docstring (missing-module-docstring)', 'tests/regrtest_data/allow_reexport/file.py:2:0: C0414: Import alias does not rename 
original package (useless-import-alias)', ...] = <built-in method split of str object at 0x5f71eb0>('\n')
E        +      where <built-in method split of str object at 0x5f71eb0> = '************* Module allow_reexport\ntests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module docstring (missing-module-docstring)\ntests/regrtest_data/allow_reexport/__init__.py:
1:0: C0414: Import alias does not rename original package (useless-import-alias)\n************* Module allow_reexport.file\ntests/regrtest_data/allow_reexport/file.py:1:0: C0114: Missing module docstring (missing-module-docstring)\ntests/regrtest_data/allow_reexport/file.
py:2:0: C0414: Import alias does not rename original package (useless-import-alias)\n'.split

tests/checkers/unittest_imports.py:155: AssertionError
__________________________________________________________________________________________________________________________ test_writing_minimal_file ___________________________________________________________________________________________________________________________

linter = Checker 'main' (responsible for 'F0001', 'F0002', 'F0010', 'F0011', 'I0001', 'I0010', 'I0011', 'I0013', 'I0020', 'I0021', 'I0022', 'E0001', 'E0011', 'W0012', 'R0022', 'E0013', 'E0014', 'E0015')
args_list = ['--accept-no-return-doc=y', 'generate', '--interactive'], reporter = None, config_file = None, verbose_mode = False

    def _config_initialization(
        linter: PyLinter,
        args_list: list[str],
        reporter: reporters.BaseReporter | reporters.MultiReporter | None = None,
        config_file: None | str | Path = None,
        verbose_mode: bool = False,
    ) -> list[str]:
        """Parse all available options, read config files and command line arguments and
        set options accordingly.
        """
        config_file = Path(config_file) if config_file else None
    
        # Set the current module to the configuration file
        # to allow raising messages on the configuration file.
        linter.set_current_module(str(config_file) if config_file else "")
    
        # Read the configuration file
        config_file_parser = _ConfigurationFileParser(verbose_mode, linter)
        try:
            config_data, config_args = config_file_parser.parse_config_file(
                file_path=config_file
            )
        except OSError as ex:
            print(ex, file=sys.stderr)
            sys.exit(32)
    
        # Run init hook, if present, before loading plugins
        if "init-hook" in config_data:
            exec(utils._unquote(config_data["init-hook"]))  # pylint: disable=exec-used
    
        # Load plugins if specified in the config file
        if "load-plugins" in config_data:
            linter.load_plugin_modules(utils._splitstrip(config_data["load-plugins"]))
    
        unrecognized_options_message = None
        # First we parse any options from a configuration file
        try:
            linter._parse_configuration_file(config_args)
        except _UnrecognizedOptionError as exc:
            unrecognized_options_message = ", ".join(exc.options)
    
        # Then, if a custom reporter is provided as argument, it may be overridden
        # by file parameters, so we re-set it here. We do this before command line
        # parsing, so it's still overridable by command line options
        if reporter:
            linter.set_reporter(reporter)
    
        # Set the current module to the command line
        # to allow raising messages on it
        linter.set_current_module("Command line")
    
        # Now we parse any options from the command line, so they can override
        # the configuration file
        parsed_args_list = linter._parse_command_line_configuration(args_list)
    
        # Remove the positional arguments separator from the list of arguments if it exists
        try:
            parsed_args_list.remove("--")
        except ValueError:
            pass
    
        # Check if there are any options that we do not recognize
        unrecognized_options: list[str] = []
        for opt in parsed_args_list:
            if opt.startswith("--"):
                unrecognized_options.append(opt[2:])
            elif opt.startswith("-"):
                unrecognized_options.append(opt[1:])
        if unrecognized_options:
            msg = ", ".join(unrecognized_options)
            try:
>               linter._arg_parser.error(f"Unrecognized option found: {msg}")

pylint/config/config_initialization.py:91: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ArgumentParser(prog='pylint-config', usage='%(prog)s [options]', description=None, formatter_class=<class 'pylint.config.help_formatter._HelpFormatter'>, conflict_handler='resolve', add_help=True), message = 'Unrecognized option found: accept-no-return-doc=y'

    def error(self, message):
        """error(message: string)
    
        Prints a usage message incorporating the message to stderr and
        exits.
    
        If you override this in a subclass, it should not return -- it
        should either exit or raise an exception.
        """
        self.print_usage(_sys.stderr)
        args = {'prog': self.prog, 'message': message}
>       self.exit(2, _('%(prog)s: error: %(message)s\n') % args)

/usr/lib/python3.11/argparse.py:2633: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ArgumentParser(prog='pylint-config', usage='%(prog)s [options]', description=None, formatter_class=<class 'pylint.config.help_formatter._HelpFormatter'>, conflict_handler='resolve', add_help=True), status = 2
message = 'pylint-config: error: Unrecognized option found: accept-no-return-doc=y\n'

    def exit(self, status=0, message=None):
        if message:
            self._print_message(message, _sys.stderr)
>       _sys.exit(status)
E       SystemExit: 2

/usr/lib/python3.11/argparse.py:2620: SystemExit

During handling of the above exception, another exception occurred:

monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f02ae3574d0>, capsys = <_pytest.capture.CaptureFixture object at 0x7f02ad923310>

    def test_writing_minimal_file(
        monkeypatch: MonkeyPatch, capsys: CaptureFixture[str]
    ) -> None:
        """Check that we can write a minimal file."""
        # Monkeypatch everything we don't want to check in this test
        monkeypatch.setattr(
            "pylint.config._pylint_config.utils.get_and_validate_format", lambda: "toml"
        )
        monkeypatch.setattr(
            "pylint.config._pylint_config.utils.get_and_validate_output_file",
            lambda: (False, Path()),
        )
    
        # Set the answers needed for the input() calls
        answers = iter(["no", "yes"])
        monkeypatch.setattr("builtins.input", lambda x: next(answers))
    
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", message="NOTE:.*", category=UserWarning)
            # Check not minimal has comments
            Run(["generate", "--interactive"], exit=False)
            captured = capsys.readouterr()
            assert any(line.startswith("#") for line in captured.out.splitlines())
    
            # Check minimal doesn't have comments and no default values
>           Run(["--accept-no-return-doc=y", "generate", "--interactive"], exit=False)

tests/config/pylint_config/test_pylint_config_generate.py:189: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
pylint/lint/run.py:167: in __init__
    args = _config_initialization(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

linter = Checker 'main' (responsible for 'F0001', 'F0002', 'F0010', 'F0011', 'I0001', 'I0010', 'I0011', 'I0013', 'I0020', 'I0021', 'I0022', 'E0001', 'E0011', 'W0012', 'R0022', 'E0013', 'E0014', 'E0015')
args_list = ['--accept-no-return-doc=y', 'generate', '--interactive'], reporter = None, config_file = None, verbose_mode = False

    def _config_initialization(
        linter: PyLinter,
        args_list: list[str],
        reporter: reporters.BaseReporter | reporters.MultiReporter | None = None,
        config_file: None | str | Path = None,
        verbose_mode: bool = False,
    ) -> list[str]:
        """Parse all available options, read config files and command line arguments and
        set options accordingly.
        """
        config_file = Path(config_file) if config_file else None
    
        # Set the current module to the configuration file
        # to allow raising messages on the configuration file.
        linter.set_current_module(str(config_file) if config_file else "")
    
        # Read the configuration file
        config_file_parser = _ConfigurationFileParser(verbose_mode, linter)
        try:
            config_data, config_args = config_file_parser.parse_config_file(
                file_path=config_file
            )
        except OSError as ex:
            print(ex, file=sys.stderr)
            sys.exit(32)
    
        # Run init hook, if present, before loading plugins
        if "init-hook" in config_data:
            exec(utils._unquote(config_data["init-hook"]))  # pylint: disable=exec-used
    
        # Load plugins if specified in the config file
        if "load-plugins" in config_data:
            linter.load_plugin_modules(utils._splitstrip(config_data["load-plugins"]))
    
        unrecognized_options_message = None
        # First we parse any options from a configuration file
        try:
            linter._parse_configuration_file(config_args)
        except _UnrecognizedOptionError as exc:
            unrecognized_options_message = ", ".join(exc.options)
    
        # Then, if a custom reporter is provided as argument, it may be overridden
        # by file parameters, so we re-set it here. We do this before command line
        # parsing, so it's still overridable by command line options
        if reporter:
            linter.set_reporter(reporter)
    
        # Set the current module to the command line
        # to allow raising messages on it
        linter.set_current_module("Command line")
    
        # Now we parse any options from the command line, so they can override
        # the configuration file
        parsed_args_list = linter._parse_command_line_configuration(args_list)
    
        # Remove the positional arguments separator from the list of arguments if it exists
        try:
            parsed_args_list.remove("--")
        except ValueError:
            pass
    
        # Check if there are any options that we do not recognize
        unrecognized_options: list[str] = []
        for opt in parsed_args_list:
            if opt.startswith("--"):
                unrecognized_options.append(opt[2:])
            elif opt.startswith("-"):
                unrecognized_options.append(opt[1:])
        if unrecognized_options:
            msg = ", ".join(unrecognized_options)
            try:
                linter._arg_parser.error(f"Unrecognized option found: {msg}")
            except SystemExit:
>               sys.exit(32)
E               SystemExit: 32

pylint/config/config_initialization.py:93: SystemExit
----------------------------------------------------------------------------------------------------------------------------- Captured stderr call -----------------------------------------------------------------------------------------------------------------------------
usage: pylint-config [options]
pylint-config: error: Unrecognized option found: accept-no-return-doc=y

------------------------------------------------------------------------------------------------------------ benchmark 'baseline': 11 tests ------------------------------------------------------------------------------------------------------------
Name (time in us)                                                Min                     Max                    Mean                 StdDev                  Median                    IQR            Outliers         OPS            Rounds  Iterations
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
test_baseline_benchmark_j1                                  401.0119 (1.0)        1,281.3341 (1.0)          472.1240 (1.0)          97.5748 (1.0)          432.2761 (1.0)          86.8381 (1.0)        143;73  2,118.0876 (1.0)        1186           1
test_baseline_benchmark_j2                               32,850.3882 (81.92)     57,897.5850 (45.19)     40,527.0591 (85.84)     8,617.1234 (88.31)     36,417.4860 (84.25)    13,310.6044 (153.28)        7;0     24.6749 (0.01)         28           1
test_baseline_benchmark_check_parallel_j2                34,464.7530 (85.94)    121,468.4041 (94.80)     50,444.5375 (106.85)   19,202.6138 (196.80)    44,329.6770 (102.55)   24,615.3791 (283.46)        4;1     19.8238 (0.01)         29           1
test_baseline_benchmark_j1_all_checks_single_file        42,777.3958 (106.67)   154,404.6130 (120.50)    53,728.3949 (113.80)   25,758.2881 (263.99)    45,163.8929 (104.48)    3,700.9249 (42.62)         1;3     18.6121 (0.01)         19           1
test_baseline_lots_of_files_j1                          103,981.0600 (259.30)   111,012.4451 (86.64)    106,559.6744 (225.70)    2,341.0303 (23.99)    105,807.8881 (244.77)    2,807.3893 (32.33)         3;0      9.3844 (0.00)          9           1
test_baseline_lots_of_files_j1_empty_checker            105,901.2711 (264.09)   107,586.7200 (83.96)    106,807.7466 (226.23)      593.0913 (6.08)     106,955.1004 (247.42)    1,003.9038 (11.56)         5;0      9.3626 (0.00)         10           1
test_baseline_benchmark_j1_all_checks_lots_of_files     109,798.0840 (273.80)   144,985.2560 (113.15)   118,819.1306 (251.67)   14,716.3723 (150.82)   113,123.9790 (261.69)   10,389.8605 (119.65)        1;1      8.4162 (0.00)          5           1
test_baseline_lots_of_files_j2                          124,854.0499 (311.35)   131,217.0441 (102.41)   127,280.1671 (269.59)    2,393.8721 (24.53)    126,201.8610 (291.95)    3,043.0433 (35.04)         3;0      7.8567 (0.00)          8           1
test_baseline_lots_of_files_j2_empty_checker            125,225.1931 (312.27)   130,042.7311 (101.49)   127,498.9153 (270.05)    1,963.6843 (20.12)    126,967.1459 (293.72)    3,603.2570 (41.49)         3;0      7.8432 (0.00)          8           1
test_baseline_benchmark_j1_single_working_checker       501,101.5639 (>1000.0)  504,342.8100 (393.61)   502,544.3916 (>1000.0)   1,505.1800 (15.43)    502,276.2520 (>1000.0)   2,842.3782 (32.73)         1;0      1.9899 (0.00)          5           1
test_baseline_benchmark_j2_single_working_checker       544,261.8409 (>1000.0)  596,846.4029 (465.80)   560,671.3220 (>1000.0)  21,532.3826 (220.68)   554,404.8601 (>1000.0)  25,974.6293 (299.12)        1;0      1.7836 (0.00)          5           1
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

Legend:
  Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.
  OPS: Operations Per Second, computed as 1 / Mean
=========================================================================================================================== short test summary info ============================================================================================================================
FAILED tests/checkers/unittest_imports.py::TestImportsChecker::test_allow_reexport_package - AssertionError: assert 7 == 5
FAILED tests/config/pylint_config/test_pylint_config_generate.py::test_writing_minimal_file - SystemExit: 32
============================================================================================== 2 failed, 1809 passed, 274 skipped, 17 deselected, 6 xfailed in 125.40s (0:02:05) ===============================================================================================
E: pybuild pybuild:388: test: plugin pyproject failed with: exit code=1: cd /build/pylint-2.16.2/.pybuild/cpython3_3.11/build; python3.11 -m pytest -vvvv -W ignore::DeprecationWarning -k 'not test_pkginfo and not test_do_not_import_files_from_local_directory and not test_import_plugin_from_local_directory_if_pythonpath_cwd and not test_can_list_directories_without_dunder_init and not test_fail_on_exit_code and not test__test_environ_pythonpath_no_arg and not test_linter_with_unpickleable_plugins_is_pickleable' {build_dir}/tests

Expected behavior

no tests failures

Pylint version

pylint-2.16.2
astroid 2.14.2
Python 3.11.2
pytest-7.2.1

OS / Environment

debian unstable

[Pierre-Sassoulas]

Thank you for opening the issue, I'm looking into it. Is the latest debian using python 3.11 ? The current test should pass but we're not ready yet (no handling of the new trystar node yet). What's the timeline for a version of pylint to be included in the next release ?

[sandrotosi]

yes, this is running using python 3.11.2;

i noticed some other tests were SKIPPED (Test cannot run with Python 3.11.2.) should these 2 be skipped as well?

i can exclude these 2 tests on my end, as i'd like to get pylint into unstable -> testing soon

[Pierre-Sassoulas]

should these 2 be skipped as well?

What is strange is that we've been testing on python 3.11 for quite some time now. I'm wondering what's different in debian unstable with python 3.11.2 vs our current CI with ubuntu-latest / python 3.11 (as defined by github).

i can exclude these 2 tests on my end, as i'd like to get pylint into unstable -> testing soon

Yeah, I think it's safe to ignore temporarily.

[DanielNoord]

The second issue seems to be caused by a faulty config. accept-no-return-doc is only supported when load-plugins = pylint.extensions.docparams is also included in the config. Might that be going wrong?

The other one is also not related to 3.12/3.11 but feels like we are picking up an unexpected config somewhere. Could you print out the full output of that test? We are asserting that the length is 5 but it would be more helpful to know what the 7 lines are, rather than only that there are 7 lines.

[sandrotosi]

The second issue seems to be caused by a faulty config. accept-no-return-doc is only supported when load-plugins = pylint.extensions.docparams is also included in the config. Might that be going wrong?

according to pytest, the only plugin loaded is benchmark-3.2.2

The other one is also not related to 3.12/3.11 but feels like we are picking up an unexpected config somewhere. Could you print out the full output of that test? We are asserting that the length is 5 but it would be more helpful to know what the 7 lines are, rather than only that there are 7 lines.

the diff was part of the original message (altho it may have been hard to find in all that output), so here it is again, hope this helps

============================================================================================================================= test session starts ==============================================================================================================================
platform linux -- Python 3.11.2, pytest-7.2.1, pluggy-1.0.0+repack
benchmark: 3.2.2 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /build/pylint-2.16.2, configfile: pyproject.toml
plugins: benchmark-3.2.2
collected 1 item                                                                                                                                                                                                                                                               

tests/checkers/unittest_imports.py F                                                                                                                                                                                                                                     [100%]

=================================================================================================================================== FAILURES ===================================================================================================================================
________________________________________________________________________________________________________________ TestImportsChecker.test_allow_reexport_package ________________________________________________________________________________________________________________

capsys = <_pytest.capture.CaptureFixture object at 0x7ff821d6a5d0>

    @staticmethod
    def test_allow_reexport_package(capsys: CaptureFixture[str]) -> None:
        """Test --allow-reexport-from-package option."""
    
        # Option disabled - useless-import-alias should always be emitted
        Run(
            [
                f"{os.path.join(REGR_DATA, 'allow_reexport')}",
                "--allow-reexport-from-package=no",
                "-sn",
            ],
            exit=False,
        )
        output, errors = capsys.readouterr()
>       assert len(output.split("\n")) == 5
E       AssertionError: assert 7 == 5
E        +  where 7 = len(['************* Module allow_reexport', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module doc...est_data/allow_reexport/file.py:2:0: C0414: Import alias does not rename original package (useless-import-alias)', ...])
E        +    where ['************* Module allow_reexport', 'tests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module doc...est_data/allow_reexport/file.py:2:0: C0414: Import alias does not rename original package (useless-import-alias)', ...] = <built-in method split of str object at 0x231b370>('\n')
E        +      where <built-in method split of str object at 0x231b370> = '************* Module allow_reexport\ntests/regrtest_data/allow_reexport/__init__.py:1:0: C0114: Missing module docstr...egrtest_data/allow_reexport/file.py:2:0: C0414: Import alias does not rename original package (useless-import-alias)\n'.split

tests/checkers/unittest_imports.py:155: AssertionError
=========================================================================================================================== short test summary info ============================================================================================================================
FAILED tests/checkers/unittest_imports.py::TestImportsChecker::test_allow_reexport_package - AssertionError: assert 7 == 5
============================================================================================================================== 1 failed in 0.28s ===============================================================================================================================