Merge pull request #98 from enpaul/enp/refactor

Overhaul for stable

.github/scripts/setup-env.sh

@@ -26,7 +26,6 @@ poetry --version --no-ansi;
 poetry run pip --version;
 
 poetry install \
-  --extras poetry \
   --quiet \
   --remove-untracked \
   --no-ansi;

.github/workflows/ci.yaml

@@ -12,8 +12,6 @@ jobs:
     strategy:
       matrix:
         python:
-          - version: "3.7"
-            toxenv: py37
           - version: "3.8"
             toxenv: py38
           - version: "3.9"
@@ -22,6 +20,8 @@ jobs:
             toxenv: py310
           - version: "3.11"
             toxenv: py311
+          - version: "3.12"
+            toxenv: py312
       fail-fast: true
     steps:
       - name: Checkout

.pre-commit-config.yaml

@@ -47,12 +47,13 @@ repos:
 
       - id: reorder-python-imports
         name: reorder-python-imports
-        entry: reorder-python-imports
+        entry: isort
         language: system
-        args:
-          - "--unclassifiable-application-module=tox_poetry_installer"
+        require_serial: true
         types:
           - python
+        args:
+          - "--filter-files"
 
       - id: black
         name: black

.pylintrc

@@ -11,7 +11,6 @@
 # --disable=W"
 disable=logging-fstring-interpolation
         ,logging-format-interpolation
-        ,bad-continuation
         ,line-too-long
         ,ungrouped-imports
         ,typecheck

pyproject.toml

@@ -23,11 +23,11 @@ classifiers = [
   "Natural Language :: English",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
 ]
 
@@ -35,30 +35,34 @@ classifiers = [
 poetry_installer = "tox_poetry_installer"
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.8"
 cleo = ">=1.0,<3.0"
 poetry = "^1.5.0"
 poetry-core = "^1.1.0"
-tox = "^4"
+tox = "^4.1"
 
 [tool.poetry.group.dev.dependencies]
-bandit = "^1.6.2"
-black = "^22.3.0"
-blacken-docs = "^1.8.0"
-ipython = {version = "^8.10.1", python = "^3.8"}
-mdformat = "^0.7"
-mdformat-gfm = "^0.3"
-mypy = "^0.930"
-pre-commit = "^2.7.1"
-pre-commit-hooks = "^3.3.0"
-pylint = "^2.13.0"
-pytest = "^6.0.2"
-pytest-cov = "^2.10.1"
-reorder-python-imports = "^2.3.5"
-safety = "^2.2.0"
-toml = "^0.10.1"
-tox = "^4"
-types-toml = "^0.10.1"
+bandit = {version = "^1.7.7", python = "^3.10"}
+black = {version = "^24.3.0", python = "^3.10"}
+blacken-docs = {version = "^1.18.0", python = "^3.10"}
+ipython = {version = "^8.10.1", python = "^3.10"}
+isort = {version = "^5.13.2", python = "^3.10"}
+mdformat = {version = "^0.7", python = "^3.10"}
+mdformat-gfm = {version = "^0.3", python = "^3.10"}
+mypy = {version = "^1.11.1", python = "^3.10"}
+pre-commit = {version = "^3.8.0", python = "^3.10"}
+pre-commit-hooks = {version = "^4.6.0", python = "^3.10"}
+pylint = {version = "^3.2.6", python = "^3.10"}
+pytest = {version = "^8.3.2", python = "^3.10"}
+pytest-cov = {version = "^5.0.0", python = "^3.10"}
+toml = {version = "^0.10.1", python = "^3.10"}
+tox = "^4.1"
+types-toml = {version = "^0.10.1", python = "^3.10"}
+
+[tool.isort]
+profile = "black"
+force_single_line = "true"
+lines_after_imports = 2
 
 [build-system]
 requires = ["poetry-core>=1.1.0"]

tests/fixtures.py

@@ -1,16 +1,16 @@
-# pylint: disable=missing-module-docstring, missing-function-docstring, unused-argument, too-few-public-methods
+# pylint: disable=missing-module-docstring,missing-function-docstring,unused-argument,too-few-public-methods,protected-access
 import time
 from pathlib import Path
 from typing import List
 
 import poetry.factory
 import poetry.installation.executor
+import poetry.installation.operations.operation
 import poetry.utils.env
 import pytest
 import tox.tox_env.python.virtual_env.runner
-from poetry.installation.operations.operation import Operation
 
-from tox_poetry_installer import utilities
+import tox_poetry_installer.hooks._tox_on_install_helpers
 
 
 TEST_PROJECT_PATH = Path(__file__).parent.resolve() / "test-project"
@@ -40,14 +40,20 @@ class MockExecutor:
     def __init__(self, env: MockVirtualEnv, **kwargs):
         self.env = env
 
-    def execute(self, operations: List[Operation]):
+    def execute(
+        self, operations: List[poetry.installation.operations.operation.Operation]
+    ):
         self.env.installed.extend([operation.package for operation in operations])
         time.sleep(1)
 
 
 @pytest.fixture
 def mock_venv(monkeypatch):
-    monkeypatch.setattr(utilities, "convert_virtualenv", lambda venv: venv)
+    monkeypatch.setattr(
+        tox_poetry_installer.hooks._tox_on_install_helpers,
+        "convert_virtualenv",
+        lambda venv: venv,
+    )
     monkeypatch.setattr(poetry.installation.executor, "Executor", MockExecutor)
     monkeypatch.setattr(
         tox.tox_env.python.virtual_env.runner, "VirtualEnvRunner", MockVirtualEnv
@@ -57,9 +63,9 @@ def mock_venv(monkeypatch):
 
 @pytest.fixture(scope="function")
 def mock_poetry_factory(monkeypatch):
-    pypoetry = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
+    project = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
 
     def mock_factory(*args, **kwargs):
-        return pypoetry
+        return project
 
     monkeypatch.setattr(poetry.factory.Factory, "create_poetry", mock_factory)

tests/test_installer.py

@@ -1,37 +1,40 @@
-# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
+# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
 import time
 from unittest import mock
 
+import poetry.factory
+import poetry.installation.executor
 import pytest
 import tox.tox_env.python.virtual_env.runner
-from poetry.factory import Factory
+
+import tox_poetry_installer.hooks._tox_on_install_helpers
 
 from .fixtures import mock_poetry_factory
 from .fixtures import mock_venv
-from tox_poetry_installer import installer
-from tox_poetry_installer import utilities
 
 
 def test_deduplication(mock_venv, mock_poetry_factory):
     """Test that the installer does not install duplicate dependencies"""
-    poetry = Factory().create_poetry(None)
-    packages: utilities.PackageMap = {
-        item.name: item for item in poetry.locker.locked_repository().packages
+    project = poetry.factory.Factory().create_poetry(None)
+    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+        item.name: item for item in project.locker.locked_repository().packages
     }
 
     venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     to_install = [packages["toml"], packages["toml"]]
 
-    installer.install(poetry, venv, to_install)
+    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+        project, venv, to_install
+    )
 
     assert len(set(to_install)) == len(venv.installed)  # pylint: disable=no-member
 
 
 def test_parallelization(mock_venv, mock_poetry_factory):
     """Test that behavior is consistent between parallel and non-parallel usage"""
-    poetry = Factory().create_poetry(None)
-    packages: utilities.PackageMap = {
-        item.name: item for item in poetry.locker.locked_repository().packages
+    project = poetry.factory.Factory().create_poetry(None)
+    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+        item.name: item for item in project.locker.locked_repository().packages
     }
 
     to_install = [
@@ -45,12 +48,16 @@ def test_parallelization(mock_venv, mock_poetry_factory):
 
     venv_sequential = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     start_sequential = time.time()
-    installer.install(poetry, venv_sequential, to_install, 0)
+    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+        project, venv_sequential, to_install, 0
+    )
     sequential = time.time() - start_sequential
 
     venv_parallel = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     start_parallel = time.time()
-    installer.install(poetry, venv_parallel, to_install, 5)
+    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+        project, venv_parallel, to_install, 5
+    )
     parallel = time.time() - start_parallel
 
     # The mock delay during package install is static (one second) so these values should all
@@ -69,22 +76,22 @@ def test_propagates_exceptions_during_installation(
 
     Regression test for https://github.com/enpaul/tox-poetry-installer/issues/86
     """
-    from tox_poetry_installer import _poetry  # pylint: disable=import-outside-toplevel
-
-    poetry = Factory().create_poetry(None)
-    packages: utilities.PackageMap = {
-        item.name: item for item in poetry.locker.locked_repository().packages
+    project = poetry.factory.Factory().create_poetry(None)
+    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+        item.name: item for item in project.locker.locked_repository().packages
     }
     to_install = [packages["toml"]]
     venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     fake_exception = ValueError("my testing exception")
 
     with mock.patch.object(
-        _poetry,
+        poetry.installation.executor,
         "Executor",
         **{"return_value.execute.side_effect": fake_exception},
     ):
         with pytest.raises(ValueError) as exc_info:
-            installer.install(poetry, venv, to_install, num_threads)
+            tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+                project, venv, to_install, num_threads
+            )
 
     assert exc_info.value is fake_exception

tests/test_metadata.py

@@ -3,6 +3,7 @@
 The next best thing to having one source of truth is having a way to ensure all of your
 sources of truth agree with each other.
 """
+
 from pathlib import Path
 
 import toml

tests/test_transients.py

@@ -1,33 +1,23 @@
-# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
+# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
 import poetry.factory
 import poetry.utils.env
 import pytest
-from poetry.puzzle.provider import Provider
+
+import tox_poetry_installer.hooks._tox_on_install_helpers
+from tox_poetry_installer import exceptions
 
 from .fixtures import mock_poetry_factory
 from .fixtures import mock_venv
-from tox_poetry_installer import constants
-from tox_poetry_installer import exceptions
-from tox_poetry_installer import utilities
-
-
-def test_exclude_unsafe():
-    """Test that the unsafe packages are properly excluded
-
-    Also ensure that the internal constant matches the value from Poetry
-    """
-    assert Provider.UNSAFE_PACKAGES == constants.UNSAFE_PACKAGES
-
-    for dep in constants.UNSAFE_PACKAGES:
-        assert not utilities.identify_transients(dep, {}, None)
 
 
 def test_allow_missing():
     """Test that the ``allow_missing`` parameter works as expected"""
     with pytest.raises(exceptions.LockedDepNotFoundError):
-        utilities.identify_transients("luke-skywalker", {}, None)
+        tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+            "luke-skywalker", {}, None
+        )
 
-    assert not utilities.identify_transients(
+    assert not tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
         "darth-vader", {}, None, allow_missing=["darth-vader"]
     )
 
@@ -47,7 +37,9 @@ def test_exclude_pep508():
         "=>foo",
     ]:
         with pytest.raises(exceptions.LockedDepVersionConflictError):
-            utilities.identify_transients(version, {}, None)
+            tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+                version, {}, None
+            )
 
 
 def test_functional(mock_poetry_factory, mock_venv):
@@ -56,8 +48,10 @@ def test_functional(mock_poetry_factory, mock_venv):
     Trivially test that it resolves dependencies properly and that the parent package
     is always the last in the returned list.
     """
-    pypoetry = poetry.factory.Factory().create_poetry(None)
-    packages = utilities.build_package_map(pypoetry)
+    project = poetry.factory.Factory().create_poetry(None)
+    packages = tox_poetry_installer.hooks._tox_on_install_helpers.build_package_map(
+        project
+    )
     venv = poetry.utils.env.VirtualEnv()  # pylint: disable=no-value-for-parameter
 
     requests_requires = [
@@ -68,12 +62,18 @@ def test_functional(mock_poetry_factory, mock_venv):
         packages["requests"][0],
     ]
 
-    transients = utilities.identify_transients("requests", packages, venv)
+    transients = tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+        "requests", packages, venv
+    )
 
     assert all((item in requests_requires) for item in transients)
     assert all((item in transients) for item in requests_requires)
 
     for package in [packages["requests"][0], packages["tox"][0], packages["flask"][0]]:
-        transients = utilities.identify_transients(package.name, packages, venv)
+        transients = (
+            tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+                package.name, packages, venv
+            )
+        )
         assert transients[-1] == package
         assert len(transients) == len(set(transients))

tox.ini

@@ -1,13 +1,11 @@
 [tox]
-envlist = py37, py38, py39, py310, py311, static, static-tests, security
+envlist = py3{8,9,10,11,12} static, static-tests, security
 skip_missing_interpreters = true
 
 [testenv]
 description = Run the tests
 require_locked_deps = true
 require_poetry = true
-extras =
-    poetry
 locked_deps =
     pytest
     pytest-cov
@@ -26,10 +24,10 @@ ignore_errors = true
 locked_deps =
     black
     blacken-docs
+    isort
     mdformat
     mdformat-gfm
     mypy
-    reorder-python-imports
     pre-commit
     pre-commit-hooks
     pylint
@@ -52,6 +50,7 @@ locked_deps =
     pylint
     pytest
     mypy
+    toml
     types-toml
 commands =
     pylint {toxinidir}/tests/ \
@@ -78,14 +77,3 @@ commands =
       --recursive \
       --quiet \
       --skip B101
-    poetry export \
-      --format requirements.txt \
-      --output {envtmpdir}/requirements.txt \
-      --without-hashes \
-      --with dev \
-      --extras poetry
-    safety check \
-      --file {envtmpdir}/requirements.txt \
-      --output text \
-      # https://github.com/pytest-dev/py/issues/287
-      --ignore 51457

tox_poetry_installer/_poetry.py

@@ -1,43 +0,0 @@
-"""You've heard of vendoirization, now get ready for internal namespace shadowing
-
-Poetry is an optional dependency of this package explicitly to support the use case of having the
-plugin and the `poetry` package installed to the same python environment; this is most common in
-containers and/or CI. In this case there are two potential problems that can arise in this case:
-
-* The installation of the plugin overwrites the installed version of Poetry resulting in
-  compatibility issues.
-* Running `poetry install --no-dev`, when this plugin is in the dev-deps, results in poetry being
-  uninstalled from the environment.
-
-To support these edge cases, and more broadly to support not messing with a system package manager,
-the `poetry` package dependency is listed as optional dependency. This allows the plugin to be
-installed to the same environment as Poetry and import that same Poetry installation here.
-
-However, simply importing Poetry on the assumption that it is installed breaks another valid use
-case: having this plugin installed alongside Tox when not using a Poetry-based project. To account
-for this the imports in this module are isolated and the resultant import error that would result
-is converted to an internal error that can be caught by callers. Rather than importing this module
-at the module scope it is imported into function scope wherever Poetry components are needed. This
-moves import errors from load time to runtime which allows the plugin to be skipped if Poetry isn't
-installed and/or a more helpful error be raised within the Tox framework.
-"""
-# pylint: disable=unused-import
-import sys
-
-from tox_poetry_installer import exceptions
-
-
-try:
-    from cleo.io.null_io import NullIO
-    from poetry.config.config import Config
-    from poetry.core.packages.dependency import Dependency as PoetryDependency
-    from poetry.core.packages.package import Package as PoetryPackage
-    from poetry.factory import Factory
-    from poetry.installation.executor import Executor
-    from poetry.installation.operations.install import Install
-    from poetry.poetry import Poetry
-    from poetry.utils.env import VirtualEnv
-except ImportError as err:
-    raise exceptions.PoetryNotInstalledError(
-        f"Failed to import a supported version of Poetry under the current environment '{sys.executable}': {err}"
-    ) from None

tox_poetry_installer/constants.py

@@ -5,7 +5,7 @@ in this module.
 
 All constants should be type hinted.
 """
-from typing import Set
+
 from typing import Tuple
 
 from tox_poetry_installer import __about__

tox_poetry_installer/exceptions.py

@@ -11,6 +11,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
    +-- LockedDepNotFoundError
    +-- ExtraNotFoundError
    +-- LockedDepsRequiredError
+   +-- LockfileParsingError
 
 """
 
@@ -41,3 +42,7 @@ class ExtraNotFoundError(ToxPoetryInstallerException):
 
 class LockedDepsRequiredError(ToxPoetryInstallerException):
     """Environment cannot specify unlocked dependencies when locked dependencies are required"""
+
+
+class LockfileParsingError(ToxPoetryInstallerException):
+    """Failed to load or parse the Poetry lockfile"""

tox_poetry_installer/hooks/_tox_on_install_helpers.py

@@ -0,0 +1,366 @@
+"""Helper functions for the :func:`tox_on_install` hook"""
+
+import collections
+import concurrent.futures
+import contextlib
+import datetime
+import pathlib
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Sequence
+from typing import Set
+
+import cleo.io.null_io
+import packaging.utils
+import poetry.config.config
+import poetry.core.packages.dependency
+import poetry.core.packages.package
+import poetry.factory
+import poetry.installation.executor
+import poetry.installation.operations.install
+import poetry.poetry
+import poetry.utils.env
+import tox.tox_env.api
+import tox.tox_env.package
+
+from tox_poetry_installer import constants
+from tox_poetry_installer import exceptions
+from tox_poetry_installer import logger
+
+
+PackageMap = Dict[str, List[poetry.core.packages.package.Package]]
+
+
+def check_preconditions(venv: tox.tox_env.api.ToxEnv) -> poetry.poetry.Poetry:
+    """Check that the local project environment meets expectations"""
+
+    # Skip running the plugin for the provisioning environment. The provisioned environment,
+    # for alternative Tox versions and/or the ``requires`` meta dependencies is specially
+    # handled by Tox and is out of scope for this plugin. Since one of the ways to install this
+    # plugin in the first place is via the Tox provisioning environment, it quickly becomes a
+    # chicken-and-egg problem.
+    if isinstance(venv, tox.tox_env.package.PackageToxEnv):
+        raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
+
+    try:
+        return poetry.factory.Factory().create_poetry(venv.core["tox_root"])
+    # Support running the plugin when the current tox project does not use Poetry for its
+    # environment/dependency management.
+    #
+    # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
+    # is) raised in many different places for different purposes.
+    except RuntimeError as err:
+        raise exceptions.SkipEnvironment(
+            f"Skipping installation of locked dependencies due to a Poetry error: {err}"
+        ) from None
+
+
+def identify_transients(
+    dep_name: str,
+    packages: PackageMap,
+    venv: poetry.utils.env.VirtualEnv,
+    allow_missing: Sequence[str] = (),
+) -> List[poetry.core.packages.package.Package]:
+    """Using a pool of packages, identify all transient dependencies of a given package name
+
+    :param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively
+                     identify the transient dependencies of
+    :param packages: All packages from the lockfile to use for identifying dependency relationships.
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param allow_missing: Sequence of package names to allow to be missing from the lockfile. Any
+                          packages that are not found in the lockfile but their name appears in this
+                          list will be silently skipped from installation.
+    :returns: List of packages that need to be installed for the requested dependency.
+
+    .. note:: The package corresponding to the dependency specified by the ``dep`` parameter will
+              be included in the returned list of packages.
+    """
+    searched: Set[str] = set()
+
+    def _transients(
+        transient: poetry.core.packages.dependency.Dependency,
+    ) -> List[poetry.core.packages.package.Package]:
+        searched.add(transient.name)
+
+        results: List[poetry.core.packages.package.Package] = []
+        for option in packages[transient.name]:
+            if venv.is_valid_for_marker(option.to_dependency().marker):
+                for requirement in option.requires:
+                    if requirement.name not in searched:
+                        results += _transients(requirement)
+                logger.debug(f"Including {option} for installation")
+                results.append(option)
+                break
+        else:
+            logger.debug(
+                f"Skipping {transient.name}: target python version is {'.'.join([str(item) for item in venv.get_version_info()])} but package requires {transient.marker}"
+            )
+
+        return results
+
+    try:
+        for option in packages[dep_name]:
+            if venv.is_valid_for_marker(option.to_dependency().marker):
+                dep = option.to_dependency()
+                break
+        else:
+            logger.warning(
+                f"Skipping {dep_name}: no locked version found compatible with target python version {'.'.join([str(item) for item in venv.get_version_info()])}"
+            )
+            return []
+
+        return _transients(dep)
+    except KeyError as err:
+        missing = err.args[0]
+
+        if missing in allow_missing:
+            logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
+            return []
+
+        if any(
+            delimiter in missing for delimiter in constants.PEP508_VERSION_DELIMITERS
+        ):
+            raise exceptions.LockedDepVersionConflictError(
+                f"Locked dependency '{missing}' cannot include version specifier"
+            ) from None
+
+        raise exceptions.LockedDepNotFoundError(
+            f"No version of locked dependency '{missing}' found in the project lockfile"
+        ) from None
+
+
+def find_project_deps(
+    packages: PackageMap,
+    venv: poetry.utils.env.VirtualEnv,
+    project: poetry.poetry.Poetry,
+    extras: Sequence[str] = (),
+) -> List[poetry.core.packages.package.Package]:
+    """Find the root project dependencies
+
+    Recursively identify the dependencies of the root project package
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param project: Poetry object for the current project
+    :param extras: Sequence of extra names to include the dependencies of
+    """
+
+    required_dep_names = [
+        item.name for item in project.package.requires if not item.is_optional()
+    ]
+
+    extra_dep_names: List[str] = []
+    for extra in extras:
+        logger.info(f"Processing project extra '{extra}'")
+        try:
+            extra_dep_names += [
+                item.name
+                for item in project.package.extras[
+                    packaging.utils.NormalizedName(extra)
+                ]
+            ]
+        except KeyError:
+            raise exceptions.ExtraNotFoundError(
+                f"Environment specifies project extra '{extra}' which was not found in the lockfile"
+            ) from None
+
+    dependencies: List[poetry.core.packages.package.Package] = []
+    for dep_name in required_dep_names + extra_dep_names:
+        dependencies += identify_transients(
+            dep_name.lower(), packages, venv, allow_missing=[project.package.name]
+        )
+
+    return dedupe_packages(dependencies)
+
+
+def find_additional_deps(
+    packages: PackageMap,
+    venv: poetry.utils.env.VirtualEnv,
+    project: poetry.poetry.Poetry,
+    dep_names: Sequence[str],
+) -> List[poetry.core.packages.package.Package]:
+    """Find additional dependencies
+
+    Recursively identify the dependencies of an arbitrary list of package names
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param project: Poetry object for the current project
+    :param dep_names: Sequence of additional dependency names to recursively find the transient
+                      dependencies for
+    """
+    dependencies: List[poetry.core.packages.package.Package] = []
+    for dep_name in dep_names:
+        dependencies += identify_transients(
+            dep_name.lower(), packages, venv, allow_missing=[project.package.name]
+        )
+
+    return dedupe_packages(dependencies)
+
+
+def find_group_deps(
+    group: str,
+    packages: PackageMap,
+    venv: poetry.utils.env.VirtualEnv,
+    project: poetry.poetry.Poetry,
+) -> List[poetry.core.packages.package.Package]:
+    """Find the dependencies belonging to a dependency group
+
+    Recursively identify the Poetry dev dependencies
+
+    :param group: Name of the dependency group from the project's ``pyproject.toml``
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param project: Poetry object for the current project
+    """
+    return find_additional_deps(
+        packages,
+        venv,
+        project,
+        # the type ignore here is due to the difficulties around getting nested data
+        # from the inherrently unstructured toml structure (which necessarily is flexibly
+        # typed) but in a situation where there is a meta-structure applied to it (i.e. a
+        # pyproject.toml structure).
+        project.pyproject.data["tool"]["poetry"]  # type: ignore
+        .get("group", {})
+        .get(group, {})
+        .get("dependencies", {})
+        .keys(),
+    )
+
+
+def find_dev_deps(
+    packages: PackageMap,
+    venv: poetry.utils.env.VirtualEnv,
+    project: poetry.poetry.Poetry,
+) -> List[poetry.core.packages.package.Package]:
+    """Find the dev dependencies
+
+    Recursively identify the Poetry dev dependencies
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param project: Poetry object for the current project
+    """
+    dev_group_deps = find_group_deps("dev", packages, venv, project)
+
+    # Legacy pyproject.toml poetry format:
+    legacy_dev_group_deps = find_additional_deps(
+        packages,
+        venv,
+        project,
+        # the type ignore here is due to the difficulties around getting nested data
+        # from the inherrently unstructured toml structure (which necessarily is flexibly
+        # typed) but in a situation where there is a meta-structure applied to it (i.e. a
+        # pyproject.toml structure).
+        project.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),  # type: ignore
+    )
+
+    # Poetry 1.2 unions these two toml sections.
+    return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
+
+
+@contextlib.contextmanager
+def _optional_parallelize(parallels: int):
+    """A bit of cheat, really
+
+    A context manager that exposes a common interface for the caller that optionally
+    enables/disables the usage of the parallel thread pooler depending on the value of
+    the ``parallels`` parameter.
+    """
+    if parallels > 0:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=parallels) as executor:
+            yield executor.submit
+    else:
+        yield lambda func, arg: func(arg)
+
+
+def install_package(
+    project: poetry.poetry.Poetry,
+    venv: tox.tox_env.api.ToxEnv,
+    packages: Collection[poetry.core.packages.package.Package],
+    parallels: int = 0,
+):
+    """Install a bunch of packages to a virtualenv
+
+    :param project: Poetry object the packages were sourced from
+    :param venv: Tox virtual environment to install the packages to
+    :param packages: List of packages to install to the virtual environment
+    :param parallels: Number of parallel processes to use for installing dependency packages, or
+                      ``None`` to disable parallelization.
+    """
+
+    logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
+
+    install_executor = poetry.installation.executor.Executor(
+        env=convert_virtualenv(venv),
+        io=cleo.io.null_io.NullIO(),
+        pool=project.pool,
+        config=poetry.config.config.Config(),
+    )
+
+    installed: Set[poetry.core.packages.package.Package] = set()
+
+    def logged_install(dependency: poetry.core.packages.package.Package) -> None:
+        start = datetime.datetime.now()
+        logger.debug(f"Installing {dependency}")
+        install_executor.execute(
+            [poetry.installation.operations.install.Install(package=dependency)]
+        )
+        end = datetime.datetime.now()
+        logger.debug(f"Finished installing {dependency} in {end - start}")
+
+    with _optional_parallelize(parallels) as executor:
+        futures = []
+        for dependency in packages:
+            if dependency not in installed:
+                installed.add(dependency)
+                logger.debug(f"Queuing {dependency}")
+                future = executor(logged_install, dependency)
+                if future is not None:
+                    futures.append(future)
+            else:
+                logger.debug(f"Skipping {dependency}, already installed")
+        logger.debug("Waiting for installs to finish...")
+
+        for future in concurrent.futures.as_completed(futures):
+            # Don't actually care about the return value, just waiting on the
+            # future to ensure any exceptions that were raised in the called
+            # function are propagated.
+            future.result()
+
+
+def dedupe_packages(
+    packages: Sequence[poetry.core.packages.package.Package],
+) -> List[poetry.core.packages.package.Package]:
+    """Deduplicates a sequence of Packages while preserving ordering
+
+    Adapted from StackOverflow: https://stackoverflow.com/a/480227
+    """
+    seen: Set[poetry.core.packages.package.Package] = set()
+    # Make this faster, avoid method lookup below
+    seen_add = seen.add
+    return [item for item in packages if not (item in seen or seen_add(item))]
+
+
+def convert_virtualenv(venv: tox.tox_env.api.ToxEnv) -> poetry.utils.env.VirtualEnv:
+    """Convert a Tox venv to a Poetry venv
+
+    :param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
+    :returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
+    """
+    return poetry.utils.env.VirtualEnv(path=pathlib.Path(venv.env_dir))
+
+
+def build_package_map(project: poetry.poetry.Poetry) -> PackageMap:
+    """Build the mapping of package names to objects
+
+    :param project: Populated poetry object to load locked packages from
+    :returns: Mapping of package names to Poetry package objects
+    """
+    packages = collections.defaultdict(list)
+    for package in project.locker.locked_repository().packages:
+        packages[str(package.name)].append(package)
+
+    return packages

tox_poetry_installer/hooks/tox_add_env_config.py

@@ -1,12 +1,16 @@
 """Add required env configuration options to the tox INI file"""
+
 from typing import List
 
-from tox.config.sets import EnvConfigSet
-from tox.plugin import impl
+import tox.config.sets
+import tox.plugin
 
 
-@impl
-def tox_add_env_config(env_conf: EnvConfigSet):
+# pylint: disable=missing-function-docstring
+@tox.plugin.impl
+def tox_add_env_config(
+    env_conf: tox.config.sets.EnvConfigSet,
+):
     env_conf.add_config(
         "poetry_dep_groups",
         of_type=List[str],

tox_poetry_installer/hooks/tox_add_option.py

@@ -1,12 +1,14 @@
 """Add additional command line arguments to tox to configure plugin behavior"""
-from tox.config.cli.parser import ToxParser
-from tox.plugin import impl
+
+import tox.config.cli.parser
+import tox.plugin
 
 from tox_poetry_installer import constants
 
 
-@impl
-def tox_add_option(parser: ToxParser):
+# pylint: disable=missing-function-docstring
+@tox.plugin.impl
+def tox_add_option(parser: tox.config.cli.parser.ToxParser):
     parser.add_argument(
         "--parallel-install-threads",
         type=int,

tox_poetry_installer/hooks/tox_on_install.py

@@ -4,40 +4,27 @@ Loads the local Poetry environment and the corresponding lockfile then pulls the
 specified by the Tox environment. Finally these dependencies are installed into the Tox
 environment using the Poetry ``PipInstaller`` backend.
 """
-import collections
-import concurrent.futures
-import contextlib
-import typing
-from datetime import datetime
-from itertools import chain
-from pathlib import Path
-from typing import Collection
-from typing import Dict
-from typing import List
-from typing import Sequence
-from typing import Set
 
-from poetry.core.packages.dependency import Dependency as PoetryDependency
-from poetry.core.packages.package import Package as PoetryPackage
-from tox.plugin import impl
-from tox.tox_env.api import ToxEnv as ToxVirtualEnv
-from tox.tox_env.package import PackageToxEnv
+import itertools
+
+import tox.plugin
+import tox.tox_env.api
 
-from tox_poetry_installer import constants
 from tox_poetry_installer import exceptions
 from tox_poetry_installer import logger
-
-if typing.TYPE_CHECKING:
-    from tox_poetry_installer import _poetry
+from tox_poetry_installer.hooks._tox_on_install_helpers import build_package_map
+from tox_poetry_installer.hooks._tox_on_install_helpers import check_preconditions
+from tox_poetry_installer.hooks._tox_on_install_helpers import convert_virtualenv
+from tox_poetry_installer.hooks._tox_on_install_helpers import dedupe_packages
+from tox_poetry_installer.hooks._tox_on_install_helpers import find_additional_deps
+from tox_poetry_installer.hooks._tox_on_install_helpers import find_group_deps
+from tox_poetry_installer.hooks._tox_on_install_helpers import find_project_deps
+from tox_poetry_installer.hooks._tox_on_install_helpers import install_package
 
 
-PackageMap = Dict[str, List[PoetryPackage]]
-
-
-@impl
-def tox_on_install(
-    tox_env: ToxVirtualEnv, section: str  # pylint: disable=unused-argument
-) -> None:
+# pylint: disable=missing-function-docstring,unused-argument
+@tox.plugin.impl
+def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
     try:
         poetry = check_preconditions(tox_env)
     except exceptions.SkipEnvironment as err:
@@ -54,10 +41,16 @@ def tox_on_install(
 
     virtualenv = convert_virtualenv(tox_env)
 
-    if not poetry.locker.is_fresh():
-        logger.warning(
-            f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
-        )
+    try:
+        if not poetry.locker.is_fresh():
+            logger.warning(
+                f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
+            )
+    except FileNotFoundError as err:
+        logger.error(f"Could not parse lockfile: {err}")
+        raise exceptions.LockfileParsingError(
+            f"Could not parse lockfile: {err}"
+        ) from err
 
     try:
         if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
@@ -69,7 +62,7 @@ def tox_on_install(
 
         group_deps = dedupe_packages(
             list(
-                chain(
+                itertools.chain(
                     *[
                         find_group_deps(group, packages, virtualenv, poetry)
                         for group in tox_env.conf["poetry_dep_groups"]
@@ -119,322 +112,3 @@ def tox_on_install(
         dependencies,
         tox_env.options.parallel_install_threads,
     )
-
-
-def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
-    """Check that the local project environment meets expectations"""
-
-    # Skip running the plugin for the provisioning environment. The provisioned environment,
-    # for alternative Tox versions and/or the ``requires`` meta dependencies is specially
-    # handled by Tox and is out of scope for this plugin. Since one of the ways to install this
-    # plugin in the first place is via the Tox provisioning environment, it quickly becomes a
-    # chicken-and-egg problem.
-    if isinstance(venv, PackageToxEnv):
-        raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
-
-    from tox_poetry_installer import _poetry
-
-    try:
-        return _poetry.Factory().create_poetry(venv.core["tox_root"])
-    # Support running the plugin when the current tox project does not use Poetry for its
-    # environment/dependency management.
-    #
-    # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
-    # is) raised in many different places for different purposes.
-    except RuntimeError as err:
-        raise exceptions.SkipEnvironment(
-            f"Skipping installation of locked dependencies due to a Poetry error: {err}"
-        ) from None
-
-
-def identify_transients(
-    dep_name: str,
-    packages: PackageMap,
-    venv: "_poetry.VirtualEnv",
-    allow_missing: Sequence[str] = (),
-) -> List[PoetryPackage]:
-    """Using a pool of packages, identify all transient dependencies of a given package name
-
-    :param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively
-                     identify the transient dependencies of
-    :param packages: All packages from the lockfile to use for identifying dependency relationships.
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param allow_missing: Sequence of package names to allow to be missing from the lockfile. Any
-                          packages that are not found in the lockfile but their name appears in this
-                          list will be silently skipped from installation.
-    :returns: List of packages that need to be installed for the requested dependency.
-
-    .. note:: The package corresponding to the dependency specified by the ``dep`` parameter will
-              be included in the returned list of packages.
-    """
-    searched: Set[str] = set()
-
-    def _transients(transient: PoetryDependency) -> List[PoetryPackage]:
-        searched.add(transient.name)
-
-        results: List[PoetryPackage] = []
-        for option in packages[transient.name]:
-            if venv.is_valid_for_marker(option.to_dependency().marker):
-                for requirement in option.requires:
-                    if requirement.name not in searched:
-                        results += _transients(requirement)
-                logger.debug(f"Including {option} for installation")
-                results.append(option)
-                break
-        else:
-            logger.debug(
-                f"Skipping {transient.name}: target python version is {'.'.join([str(item) for item in venv.get_version_info()])} but package requires {transient.marker}"
-            )
-
-        return results
-
-    try:
-        for option in packages[dep_name]:
-            if venv.is_valid_for_marker(option.to_dependency().marker):
-                dep = option.to_dependency()
-                break
-        else:
-            logger.warning(
-                f"Skipping {dep_name}: no locked version found compatible with target python version {'.'.join([str(item) for item in venv.get_version_info()])}"
-            )
-            return []
-
-        return _transients(dep)
-    except KeyError as err:
-        missing = err.args[0]
-
-        if missing in allow_missing:
-            logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
-            return []
-
-        if any(
-            delimiter in missing for delimiter in constants.PEP508_VERSION_DELIMITERS
-        ):
-            raise exceptions.LockedDepVersionConflictError(
-                f"Locked dependency '{missing}' cannot include version specifier"
-            ) from None
-
-        raise exceptions.LockedDepNotFoundError(
-            f"No version of locked dependency '{missing}' found in the project lockfile"
-        ) from None
-
-
-def find_project_deps(
-    packages: PackageMap,
-    venv: "_poetry.VirtualEnv",
-    poetry: "_poetry.Poetry",
-    extras: Sequence[str] = (),
-) -> List[PoetryPackage]:
-    """Find the root project dependencies
-
-    Recursively identify the dependencies of the root project package
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param poetry: Poetry object for the current project
-    :param extras: Sequence of extra names to include the dependencies of
-    """
-
-    required_dep_names = [
-        item.name for item in poetry.package.requires if not item.is_optional()
-    ]
-
-    extra_dep_names: List[str] = []
-    for extra in extras:
-        logger.info(f"Processing project extra '{extra}'")
-        try:
-            extra_dep_names += [item.name for item in poetry.package.extras[extra]]
-        except KeyError:
-            raise exceptions.ExtraNotFoundError(
-                f"Environment specifies project extra '{extra}' which was not found in the lockfile"
-            ) from None
-
-    dependencies: List[PoetryPackage] = []
-    for dep_name in required_dep_names + extra_dep_names:
-        dependencies += identify_transients(
-            dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
-        )
-
-    return dedupe_packages(dependencies)
-
-
-def find_additional_deps(
-    packages: PackageMap,
-    venv: "_poetry.VirtualEnv",
-    poetry: "_poetry.Poetry",
-    dep_names: Sequence[str],
-) -> List[PoetryPackage]:
-    """Find additional dependencies
-
-    Recursively identify the dependencies of an arbitrary list of package names
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param poetry: Poetry object for the current project
-    :param dep_names: Sequence of additional dependency names to recursively find the transient
-                      dependencies for
-    """
-    dependencies: List[PoetryPackage] = []
-    for dep_name in dep_names:
-        dependencies += identify_transients(
-            dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
-        )
-
-    return dedupe_packages(dependencies)
-
-
-def find_group_deps(
-    group: str,
-    packages: PackageMap,
-    venv: "_poetry.VirtualEnv",
-    poetry: "_poetry.Poetry",
-) -> List[PoetryPackage]:
-    """Find the dependencies belonging to a dependency group
-
-    Recursively identify the Poetry dev dependencies
-
-    :param group: Name of the dependency group from the project's ``pyproject.toml``
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param poetry: Poetry object for the current project
-    """
-    return find_additional_deps(
-        packages,
-        venv,
-        poetry,
-        poetry.pyproject.data["tool"]["poetry"]
-        .get("group", {})
-        .get(group, {})
-        .get("dependencies", {})
-        .keys(),
-    )
-
-
-def find_dev_deps(
-    packages: PackageMap, venv: "_poetry.VirtualEnv", poetry: "_poetry.Poetry"
-) -> List[PoetryPackage]:
-    """Find the dev dependencies
-
-    Recursively identify the Poetry dev dependencies
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param poetry: Poetry object for the current project
-    """
-    dev_group_deps = find_group_deps("dev", packages, venv, poetry)
-
-    # Legacy pyproject.toml poetry format:
-    legacy_dev_group_deps = find_additional_deps(
-        packages,
-        venv,
-        poetry,
-        poetry.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),
-    )
-
-    # Poetry 1.2 unions these two toml sections.
-    return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
-
-
-def install_package(
-    poetry: "_poetry.Poetry",
-    venv: ToxVirtualEnv,
-    packages: Collection["_poetry.PoetryPackage"],
-    parallels: int = 0,
-):
-    """Install a bunch of packages to a virtualenv
-
-    :param poetry: Poetry object the packages were sourced from
-    :param venv: Tox virtual environment to install the packages to
-    :param packages: List of packages to install to the virtual environment
-    :param parallels: Number of parallel processes to use for installing dependency packages, or
-                      ``None`` to disable parallelization.
-    """
-    from tox_poetry_installer import _poetry
-
-    logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
-
-    install_executor = _poetry.Executor(
-        env=convert_virtualenv(venv),
-        io=_poetry.NullIO(),
-        pool=poetry.pool,
-        config=_poetry.Config(),
-    )
-
-    installed: Set[_poetry.PoetryPackage] = set()
-
-    def logged_install(dependency: _poetry.PoetryPackage) -> None:
-        start = datetime.now()
-        logger.debug(f"Installing {dependency}")
-        install_executor.execute([_poetry.Install(package=dependency)])
-        end = datetime.now()
-        logger.debug(f"Finished installing {dependency} in {end - start}")
-
-    @contextlib.contextmanager
-    def _optional_parallelize():
-        """A bit of cheat, really
-
-        A context manager that exposes a common interface for the caller that optionally
-        enables/disables the usage of the parallel thread pooler depending on the value of
-        the ``parallels`` parameter.
-        """
-        if parallels > 0:
-            with concurrent.futures.ThreadPoolExecutor(
-                max_workers=parallels
-            ) as executor:
-                yield executor.submit
-        else:
-            yield lambda func, arg: func(arg)
-
-    with _optional_parallelize() as executor:
-        futures = []
-        for dependency in packages:
-            if dependency not in installed:
-                installed.add(dependency)
-                logger.debug(f"Queuing {dependency}")
-                future = executor(logged_install, dependency)
-                if future is not None:
-                    futures.append(future)
-            else:
-                logger.debug(f"Skipping {dependency}, already installed")
-        logger.debug("Waiting for installs to finish...")
-
-        for future in concurrent.futures.as_completed(futures):
-            # Don't actually care about the return value, just waiting on the
-            # future to ensure any exceptions that were raised in the called
-            # function are propagated.
-            future.result()
-
-
-def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
-    """Deduplicates a sequence of PoetryPackages while preserving ordering
-
-    Adapted from StackOverflow: https://stackoverflow.com/a/480227
-    """
-    seen: Set[PoetryPackage] = set()
-    # Make this faster, avoid method lookup below
-    seen_add = seen.add
-    return [p for p in packages if not (p in seen or seen_add(p))]
-
-
-def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv":
-    """Convert a Tox venv to a Poetry venv
-
-    :param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
-    :returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
-    """
-    from tox_poetry_installer import _poetry
-
-    return _poetry.VirtualEnv(path=Path(venv.env_dir))
-
-
-def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
-    """Build the mapping of package names to objects
-
-    :param poetry: Populated poetry object to load locked packages from
-    :returns: Mapping of package names to Poetry package objects
-    """
-    packages = collections.defaultdict(list)
-    for package in poetry.locker.locked_repository().packages:
-        packages[package.name].append(package)
-
-    return packages

tox_poetry_installer/logger.py

@@ -4,6 +4,7 @@ Calling ``tox.reporter.something()`` and having to format a string with the pref
 gets really old fast, but more importantly it also makes the flow of the main code
 more difficult to follow because of the added complexity.
 """
+
 import logging
 
 from tox_poetry_installer import constants