Replace optional [poetry] extra with explicit poetry dependencies

Fixes #79

.github/scripts/setup-env.sh

@@ -26,6 +26,7 @@ poetry --version --no-ansi;
 poetry run pip --version;
 
 poetry install \
+  --extras poetry \
   --quiet \
   --remove-untracked \
   --no-ansi;

.github/workflows/ci.yaml

@@ -12,6 +12,8 @@ jobs:
     strategy:
       matrix:
         python:
+          - version: "3.7"
+            toxenv: py37
           - version: "3.8"
             toxenv: py38
           - version: "3.9"
@@ -20,8 +22,6 @@ jobs:
             toxenv: py310
           - version: "3.11"
             toxenv: py311
-          - version: "3.12"
-            toxenv: py312
       fail-fast: true
     steps:
       - name: Checkout

.pre-commit-config.yaml

@@ -47,13 +47,12 @@ repos:
 
       - id: reorder-python-imports
         name: reorder-python-imports
-        entry: isort
+        entry: reorder-python-imports
         language: system
-        require_serial: true
+        args:
+          - "--unclassifiable-application-module=tox_poetry_installer"
         types:
           - python
-        args:
-          - "--filter-files"
 
       - id: black
         name: black

.pylintrc

@@ -11,6 +11,7 @@
 # --disable=W"
 disable=logging-fstring-interpolation
         ,logging-format-interpolation
+        ,bad-continuation
         ,line-too-long
         ,ungrouped-imports
         ,typecheck

pyproject.toml

@@ -23,11 +23,11 @@ classifiers = [
   "Natural Language :: English",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
 ]
 
@@ -35,34 +35,30 @@ classifiers = [
 poetry_installer = "tox_poetry_installer"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.7"
 cleo = ">=1.0,<3.0"
 poetry = "^1.5.0"
 poetry-core = "^1.1.0"
-tox = "^4.1"
+tox = "^4"
 
 [tool.poetry.group.dev.dependencies]
-bandit = {version = "^1.7.7", python = "^3.10"}
+bandit = "^1.6.2"
-black = {version = "^24.3.0", python = "^3.10"}
+black = "^22.3.0"
-blacken-docs = {version = "^1.18.0", python = "^3.10"}
+blacken-docs = "^1.8.0"
-ipython = {version = "^8.10.1", python = "^3.10"}
+ipython = {version = "^8.10.1", python = "^3.8"}
-isort = {version = "^5.13.2", python = "^3.10"}
+mdformat = "^0.7"
-mdformat = {version = "^0.7", python = "^3.10"}
+mdformat-gfm = "^0.3"
-mdformat-gfm = {version = "^0.3", python = "^3.10"}
+mypy = "^0.930"
-mypy = {version = "^1.11.1", python = "^3.10"}
+pre-commit = "^2.7.1"
-pre-commit = {version = "^3.8.0", python = "^3.10"}
+pre-commit-hooks = "^3.3.0"
-pre-commit-hooks = {version = "^4.6.0", python = "^3.10"}
+pylint = "^2.13.0"
-pylint = {version = "^3.2.6", python = "^3.10"}
+pytest = "^6.0.2"
-pytest = {version = "^8.3.2", python = "^3.10"}
+pytest-cov = "^2.10.1"
-pytest-cov = {version = "^5.0.0", python = "^3.10"}
+reorder-python-imports = "^2.3.5"
-toml = {version = "^0.10.1", python = "^3.10"}
+safety = "^2.2.0"
-tox = "^4.1"
+toml = "^0.10.1"
-types-toml = {version = "^0.10.1", python = "^3.10"}
+tox = "^4"
-
+types-toml = "^0.10.1"
-[tool.isort]
-profile = "black"
-force_single_line = "true"
-lines_after_imports = 2
 
 [build-system]
 requires = ["poetry-core>=1.1.0"]

tests/fixtures.py

@@ -1,16 +1,16 @@
-# pylint: disable=missing-module-docstring,missing-function-docstring,unused-argument,too-few-public-methods,protected-access
+# pylint: disable=missing-module-docstring, missing-function-docstring, unused-argument, too-few-public-methods
 import time
 from pathlib import Path
 from typing import List
 
 import poetry.factory
 import poetry.installation.executor
-import poetry.installation.operations.operation
 import poetry.utils.env
 import pytest
 import tox.tox_env.python.virtual_env.runner
+from poetry.installation.operations.operation import Operation
 
-import tox_poetry_installer.hooks._tox_on_install_helpers
+from tox_poetry_installer import utilities
 
 
 TEST_PROJECT_PATH = Path(__file__).parent.resolve() / "test-project"
@@ -40,20 +40,14 @@ class MockExecutor:
     def __init__(self, env: MockVirtualEnv, **kwargs):
         self.env = env
 
-    def execute(
+    def execute(self, operations: List[Operation]):
-        self, operations: List[poetry.installation.operations.operation.Operation]
-    ):
         self.env.installed.extend([operation.package for operation in operations])
         time.sleep(1)
 
 
 @pytest.fixture
 def mock_venv(monkeypatch):
-    monkeypatch.setattr(
+    monkeypatch.setattr(utilities, "convert_virtualenv", lambda venv: venv)
-        tox_poetry_installer.hooks._tox_on_install_helpers,
-        "convert_virtualenv",
-        lambda venv: venv,
-    )
     monkeypatch.setattr(poetry.installation.executor, "Executor", MockExecutor)
     monkeypatch.setattr(
         tox.tox_env.python.virtual_env.runner, "VirtualEnvRunner", MockVirtualEnv
@@ -63,9 +57,9 @@ def mock_venv(monkeypatch):
 
 @pytest.fixture(scope="function")
 def mock_poetry_factory(monkeypatch):
-    project = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
+    pypoetry = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
 
     def mock_factory(*args, **kwargs):
-        return project
+        return pypoetry
 
     monkeypatch.setattr(poetry.factory.Factory, "create_poetry", mock_factory)

tests/test_installer.py

@@ -1,40 +1,37 @@
-# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
+# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
 import time
 from unittest import mock
 
-import poetry.factory
-import poetry.installation.executor
 import pytest
 import tox.tox_env.python.virtual_env.runner
-
+from poetry.factory import Factory
-import tox_poetry_installer.hooks._tox_on_install_helpers
 
 from .fixtures import mock_poetry_factory
 from .fixtures import mock_venv
+from tox_poetry_installer import installer
+from tox_poetry_installer import utilities
 
 
 def test_deduplication(mock_venv, mock_poetry_factory):
     """Test that the installer does not install duplicate dependencies"""
-    project = poetry.factory.Factory().create_poetry(None)
+    poetry = Factory().create_poetry(None)
-    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+    packages: utilities.PackageMap = {
-        item.name: item for item in project.locker.locked_repository().packages
+        item.name: item for item in poetry.locker.locked_repository().packages
     }
 
     venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     to_install = [packages["toml"], packages["toml"]]
 
-    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+    installer.install(poetry, venv, to_install)
-        project, venv, to_install
-    )
 
     assert len(set(to_install)) == len(venv.installed)  # pylint: disable=no-member
 
 
 def test_parallelization(mock_venv, mock_poetry_factory):
     """Test that behavior is consistent between parallel and non-parallel usage"""
-    project = poetry.factory.Factory().create_poetry(None)
+    poetry = Factory().create_poetry(None)
-    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+    packages: utilities.PackageMap = {
-        item.name: item for item in project.locker.locked_repository().packages
+        item.name: item for item in poetry.locker.locked_repository().packages
     }
 
     to_install = [
@@ -48,16 +45,12 @@ def test_parallelization(mock_venv, mock_poetry_factory):
 
     venv_sequential = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     start_sequential = time.time()
-    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+    installer.install(poetry, venv_sequential, to_install, 0)
-        project, venv_sequential, to_install, 0
-    )
     sequential = time.time() - start_sequential
 
     venv_parallel = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     start_parallel = time.time()
-    tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+    installer.install(poetry, venv_parallel, to_install, 5)
-        project, venv_parallel, to_install, 5
-    )
     parallel = time.time() - start_parallel
 
     # The mock delay during package install is static (one second) so these values should all
@@ -76,22 +69,22 @@ def test_propagates_exceptions_during_installation(
 
     Regression test for https://github.com/enpaul/tox-poetry-installer/issues/86
     """
-    project = poetry.factory.Factory().create_poetry(None)
+    from tox_poetry_installer import _poetry  # pylint: disable=import-outside-toplevel
-    packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
+
-        item.name: item for item in project.locker.locked_repository().packages
+    poetry = Factory().create_poetry(None)
+    packages: utilities.PackageMap = {
+        item.name: item for item in poetry.locker.locked_repository().packages
     }
     to_install = [packages["toml"]]
     venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
     fake_exception = ValueError("my testing exception")
 
     with mock.patch.object(
-        poetry.installation.executor,
+        _poetry,
         "Executor",
         **{"return_value.execute.side_effect": fake_exception},
     ):
         with pytest.raises(ValueError) as exc_info:
-            tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
+            installer.install(poetry, venv, to_install, num_threads)
-                project, venv, to_install, num_threads
-            )
 
     assert exc_info.value is fake_exception

tests/test_metadata.py

@@ -3,7 +3,6 @@
 The next best thing to having one source of truth is having a way to ensure all of your
 sources of truth agree with each other.
 """
-
 from pathlib import Path
 
 import toml

tests/test_transients.py

@@ -1,23 +1,33 @@
-# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
+# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
 import poetry.factory
 import poetry.utils.env
 import pytest
-
+from poetry.puzzle.provider import Provider
-import tox_poetry_installer.hooks._tox_on_install_helpers
-from tox_poetry_installer import exceptions
 
 from .fixtures import mock_poetry_factory
 from .fixtures import mock_venv
+from tox_poetry_installer import constants
+from tox_poetry_installer import exceptions
+from tox_poetry_installer import utilities
+
+
+def test_exclude_unsafe():
+    """Test that the unsafe packages are properly excluded
+
+    Also ensure that the internal constant matches the value from Poetry
+    """
+    assert Provider.UNSAFE_PACKAGES == constants.UNSAFE_PACKAGES
+
+    for dep in constants.UNSAFE_PACKAGES:
+        assert not utilities.identify_transients(dep, {}, None)
 
 
 def test_allow_missing():
     """Test that the ``allow_missing`` parameter works as expected"""
     with pytest.raises(exceptions.LockedDepNotFoundError):
-        tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+        utilities.identify_transients("luke-skywalker", {}, None)
-            "luke-skywalker", {}, None
-        )
 
-    assert not tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+    assert not utilities.identify_transients(
         "darth-vader", {}, None, allow_missing=["darth-vader"]
     )
 
@@ -37,9 +47,7 @@ def test_exclude_pep508():
         "=>foo",
     ]:
         with pytest.raises(exceptions.LockedDepVersionConflictError):
-            tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+            utilities.identify_transients(version, {}, None)
-                version, {}, None
-            )
 
 
 def test_functional(mock_poetry_factory, mock_venv):
@@ -48,10 +56,8 @@ def test_functional(mock_poetry_factory, mock_venv):
     Trivially test that it resolves dependencies properly and that the parent package
     is always the last in the returned list.
     """
-    project = poetry.factory.Factory().create_poetry(None)
+    pypoetry = poetry.factory.Factory().create_poetry(None)
-    packages = tox_poetry_installer.hooks._tox_on_install_helpers.build_package_map(
+    packages = utilities.build_package_map(pypoetry)
-        project
-    )
     venv = poetry.utils.env.VirtualEnv()  # pylint: disable=no-value-for-parameter
 
     requests_requires = [
@@ -62,18 +68,12 @@ def test_functional(mock_poetry_factory, mock_venv):
         packages["requests"][0],
     ]
 
-    transients = tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
+    transients = utilities.identify_transients("requests", packages, venv)
-        "requests", packages, venv
-    )
 
     assert all((item in requests_requires) for item in transients)
     assert all((item in transients) for item in requests_requires)
 
     for package in [packages["requests"][0], packages["tox"][0], packages["flask"][0]]:
-        transients = (
+        transients = utilities.identify_transients(package.name, packages, venv)
-            tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
-                package.name, packages, venv
-            )
-        )
         assert transients[-1] == package
         assert len(transients) == len(set(transients))

tox.ini

@@ -1,11 +1,13 @@
 [tox]
-envlist = py3{8,9,10,11,12} static, static-tests, security
+envlist = py37, py38, py39, py310, py311, static, static-tests, security
 skip_missing_interpreters = true
 
 [testenv]
 description = Run the tests
 require_locked_deps = true
 require_poetry = true
+extras =
+    poetry
 locked_deps =
     pytest
     pytest-cov
@@ -24,10 +26,10 @@ ignore_errors = true
 locked_deps =
     black
     blacken-docs
-    isort
     mdformat
     mdformat-gfm
     mypy
+    reorder-python-imports
     pre-commit
     pre-commit-hooks
     pylint
@@ -50,7 +52,6 @@ locked_deps =
     pylint
     pytest
     mypy
-    toml
     types-toml
 commands =
     pylint {toxinidir}/tests/ \
@@ -77,3 +78,14 @@ commands =
       --recursive \
       --quiet \
       --skip B101
+    poetry export \
+      --format requirements.txt \
+      --output {envtmpdir}/requirements.txt \
+      --without-hashes \
+      --with dev \
+      --extras poetry
+    safety check \
+      --file {envtmpdir}/requirements.txt \
+      --output text \
+      # https://github.com/pytest-dev/py/issues/287
+      --ignore 51457

tox_poetry_installer/_poetry.py

@@ -0,0 +1,43 @@
+"""You've heard of vendoirization, now get ready for internal namespace shadowing
+
+Poetry is an optional dependency of this package explicitly to support the use case of having the
+plugin and the `poetry` package installed to the same python environment; this is most common in
+containers and/or CI. In this case there are two potential problems that can arise in this case:
+
+* The installation of the plugin overwrites the installed version of Poetry resulting in
+  compatibility issues.
+* Running `poetry install --no-dev`, when this plugin is in the dev-deps, results in poetry being
+  uninstalled from the environment.
+
+To support these edge cases, and more broadly to support not messing with a system package manager,
+the `poetry` package dependency is listed as optional dependency. This allows the plugin to be
+installed to the same environment as Poetry and import that same Poetry installation here.
+
+However, simply importing Poetry on the assumption that it is installed breaks another valid use
+case: having this plugin installed alongside Tox when not using a Poetry-based project. To account
+for this the imports in this module are isolated and the resultant import error that would result
+is converted to an internal error that can be caught by callers. Rather than importing this module
+at the module scope it is imported into function scope wherever Poetry components are needed. This
+moves import errors from load time to runtime which allows the plugin to be skipped if Poetry isn't
+installed and/or a more helpful error be raised within the Tox framework.
+"""
+# pylint: disable=unused-import
+import sys
+
+from tox_poetry_installer import exceptions
+
+
+try:
+    from cleo.io.null_io import NullIO
+    from poetry.config.config import Config
+    from poetry.core.packages.dependency import Dependency as PoetryDependency
+    from poetry.core.packages.package import Package as PoetryPackage
+    from poetry.factory import Factory
+    from poetry.installation.executor import Executor
+    from poetry.installation.operations.install import Install
+    from poetry.poetry import Poetry
+    from poetry.utils.env import VirtualEnv
+except ImportError as err:
+    raise exceptions.PoetryNotInstalledError(
+        f"Failed to import a supported version of Poetry under the current environment '{sys.executable}': {err}"
+    ) from None

tox_poetry_installer/constants.py

@@ -5,7 +5,7 @@ in this module.
 
 All constants should be type hinted.
 """
-
+from typing import Set
 from typing import Tuple
 
 from tox_poetry_installer import __about__

tox_poetry_installer/exceptions.py

@@ -11,7 +11,6 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
    +-- LockedDepNotFoundError
    +-- ExtraNotFoundError
    +-- LockedDepsRequiredError
-   +-- LockfileParsingError
 
 """
 
@@ -42,7 +41,3 @@ class ExtraNotFoundError(ToxPoetryInstallerException):
 
 class LockedDepsRequiredError(ToxPoetryInstallerException):
     """Environment cannot specify unlocked dependencies when locked dependencies are required"""
-
-
-class LockfileParsingError(ToxPoetryInstallerException):
-    """Failed to load or parse the Poetry lockfile"""

tox_poetry_installer/hooks/_tox_on_install_helpers.py

@@ -1,366 +0,0 @@
-"""Helper functions for the :func:`tox_on_install` hook"""
-
-import collections
-import concurrent.futures
-import contextlib
-import datetime
-import pathlib
-from typing import Collection
-from typing import Dict
-from typing import List
-from typing import Sequence
-from typing import Set
-
-import cleo.io.null_io
-import packaging.utils
-import poetry.config.config
-import poetry.core.packages.dependency
-import poetry.core.packages.package
-import poetry.factory
-import poetry.installation.executor
-import poetry.installation.operations.install
-import poetry.poetry
-import poetry.utils.env
-import tox.tox_env.api
-import tox.tox_env.package
-
-from tox_poetry_installer import constants
-from tox_poetry_installer import exceptions
-from tox_poetry_installer import logger
-
-
-PackageMap = Dict[str, List[poetry.core.packages.package.Package]]
-
-
-def check_preconditions(venv: tox.tox_env.api.ToxEnv) -> poetry.poetry.Poetry:
-    """Check that the local project environment meets expectations"""
-
-    # Skip running the plugin for the provisioning environment. The provisioned environment,
-    # for alternative Tox versions and/or the ``requires`` meta dependencies is specially
-    # handled by Tox and is out of scope for this plugin. Since one of the ways to install this
-    # plugin in the first place is via the Tox provisioning environment, it quickly becomes a
-    # chicken-and-egg problem.
-    if isinstance(venv, tox.tox_env.package.PackageToxEnv):
-        raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
-
-    try:
-        return poetry.factory.Factory().create_poetry(venv.core["tox_root"])
-    # Support running the plugin when the current tox project does not use Poetry for its
-    # environment/dependency management.
-    #
-    # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
-    # is) raised in many different places for different purposes.
-    except RuntimeError as err:
-        raise exceptions.SkipEnvironment(
-            f"Skipping installation of locked dependencies due to a Poetry error: {err}"
-        ) from None
-
-
-def identify_transients(
-    dep_name: str,
-    packages: PackageMap,
-    venv: poetry.utils.env.VirtualEnv,
-    allow_missing: Sequence[str] = (),
-) -> List[poetry.core.packages.package.Package]:
-    """Using a pool of packages, identify all transient dependencies of a given package name
-
-    :param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively
-                     identify the transient dependencies of
-    :param packages: All packages from the lockfile to use for identifying dependency relationships.
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param allow_missing: Sequence of package names to allow to be missing from the lockfile. Any
-                          packages that are not found in the lockfile but their name appears in this
-                          list will be silently skipped from installation.
-    :returns: List of packages that need to be installed for the requested dependency.
-
-    .. note:: The package corresponding to the dependency specified by the ``dep`` parameter will
-              be included in the returned list of packages.
-    """
-    searched: Set[str] = set()
-
-    def _transients(
-        transient: poetry.core.packages.dependency.Dependency,
-    ) -> List[poetry.core.packages.package.Package]:
-        searched.add(transient.name)
-
-        results: List[poetry.core.packages.package.Package] = []
-        for option in packages[transient.name]:
-            if venv.is_valid_for_marker(option.to_dependency().marker):
-                for requirement in option.requires:
-                    if requirement.name not in searched:
-                        results += _transients(requirement)
-                logger.debug(f"Including {option} for installation")
-                results.append(option)
-                break
-        else:
-            logger.debug(
-                f"Skipping {transient.name}: target python version is {'.'.join([str(item) for item in venv.get_version_info()])} but package requires {transient.marker}"
-            )
-
-        return results
-
-    try:
-        for option in packages[dep_name]:
-            if venv.is_valid_for_marker(option.to_dependency().marker):
-                dep = option.to_dependency()
-                break
-        else:
-            logger.warning(
-                f"Skipping {dep_name}: no locked version found compatible with target python version {'.'.join([str(item) for item in venv.get_version_info()])}"
-            )
-            return []
-
-        return _transients(dep)
-    except KeyError as err:
-        missing = err.args[0]
-
-        if missing in allow_missing:
-            logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
-            return []
-
-        if any(
-            delimiter in missing for delimiter in constants.PEP508_VERSION_DELIMITERS
-        ):
-            raise exceptions.LockedDepVersionConflictError(
-                f"Locked dependency '{missing}' cannot include version specifier"
-            ) from None
-
-        raise exceptions.LockedDepNotFoundError(
-            f"No version of locked dependency '{missing}' found in the project lockfile"
-        ) from None
-
-
-def find_project_deps(
-    packages: PackageMap,
-    venv: poetry.utils.env.VirtualEnv,
-    project: poetry.poetry.Poetry,
-    extras: Sequence[str] = (),
-) -> List[poetry.core.packages.package.Package]:
-    """Find the root project dependencies
-
-    Recursively identify the dependencies of the root project package
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param project: Poetry object for the current project
-    :param extras: Sequence of extra names to include the dependencies of
-    """
-
-    required_dep_names = [
-        item.name for item in project.package.requires if not item.is_optional()
-    ]
-
-    extra_dep_names: List[str] = []
-    for extra in extras:
-        logger.info(f"Processing project extra '{extra}'")
-        try:
-            extra_dep_names += [
-                item.name
-                for item in project.package.extras[
-                    packaging.utils.NormalizedName(extra)
-                ]
-            ]
-        except KeyError:
-            raise exceptions.ExtraNotFoundError(
-                f"Environment specifies project extra '{extra}' which was not found in the lockfile"
-            ) from None
-
-    dependencies: List[poetry.core.packages.package.Package] = []
-    for dep_name in required_dep_names + extra_dep_names:
-        dependencies += identify_transients(
-            dep_name.lower(), packages, venv, allow_missing=[project.package.name]
-        )
-
-    return dedupe_packages(dependencies)
-
-
-def find_additional_deps(
-    packages: PackageMap,
-    venv: poetry.utils.env.VirtualEnv,
-    project: poetry.poetry.Poetry,
-    dep_names: Sequence[str],
-) -> List[poetry.core.packages.package.Package]:
-    """Find additional dependencies
-
-    Recursively identify the dependencies of an arbitrary list of package names
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param project: Poetry object for the current project
-    :param dep_names: Sequence of additional dependency names to recursively find the transient
-                      dependencies for
-    """
-    dependencies: List[poetry.core.packages.package.Package] = []
-    for dep_name in dep_names:
-        dependencies += identify_transients(
-            dep_name.lower(), packages, venv, allow_missing=[project.package.name]
-        )
-
-    return dedupe_packages(dependencies)
-
-
-def find_group_deps(
-    group: str,
-    packages: PackageMap,
-    venv: poetry.utils.env.VirtualEnv,
-    project: poetry.poetry.Poetry,
-) -> List[poetry.core.packages.package.Package]:
-    """Find the dependencies belonging to a dependency group
-
-    Recursively identify the Poetry dev dependencies
-
-    :param group: Name of the dependency group from the project's ``pyproject.toml``
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param project: Poetry object for the current project
-    """
-    return find_additional_deps(
-        packages,
-        venv,
-        project,
-        # the type ignore here is due to the difficulties around getting nested data
-        # from the inherrently unstructured toml structure (which necessarily is flexibly
-        # typed) but in a situation where there is a meta-structure applied to it (i.e. a
-        # pyproject.toml structure).
-        project.pyproject.data["tool"]["poetry"]  # type: ignore
-        .get("group", {})
-        .get(group, {})
-        .get("dependencies", {})
-        .keys(),
-    )
-
-
-def find_dev_deps(
-    packages: PackageMap,
-    venv: poetry.utils.env.VirtualEnv,
-    project: poetry.poetry.Poetry,
-) -> List[poetry.core.packages.package.Package]:
-    """Find the dev dependencies
-
-    Recursively identify the Poetry dev dependencies
-
-    :param packages: Mapping of all locked package names to their corresponding package object
-    :param venv: Poetry virtual environment to use for package compatibility checks
-    :param project: Poetry object for the current project
-    """
-    dev_group_deps = find_group_deps("dev", packages, venv, project)
-
-    # Legacy pyproject.toml poetry format:
-    legacy_dev_group_deps = find_additional_deps(
-        packages,
-        venv,
-        project,
-        # the type ignore here is due to the difficulties around getting nested data
-        # from the inherrently unstructured toml structure (which necessarily is flexibly
-        # typed) but in a situation where there is a meta-structure applied to it (i.e. a
-        # pyproject.toml structure).
-        project.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),  # type: ignore
-    )
-
-    # Poetry 1.2 unions these two toml sections.
-    return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
-
-
-@contextlib.contextmanager
-def _optional_parallelize(parallels: int):
-    """A bit of cheat, really
-
-    A context manager that exposes a common interface for the caller that optionally
-    enables/disables the usage of the parallel thread pooler depending on the value of
-    the ``parallels`` parameter.
-    """
-    if parallels > 0:
-        with concurrent.futures.ThreadPoolExecutor(max_workers=parallels) as executor:
-            yield executor.submit
-    else:
-        yield lambda func, arg: func(arg)
-
-
-def install_package(
-    project: poetry.poetry.Poetry,
-    venv: tox.tox_env.api.ToxEnv,
-    packages: Collection[poetry.core.packages.package.Package],
-    parallels: int = 0,
-):
-    """Install a bunch of packages to a virtualenv
-
-    :param project: Poetry object the packages were sourced from
-    :param venv: Tox virtual environment to install the packages to
-    :param packages: List of packages to install to the virtual environment
-    :param parallels: Number of parallel processes to use for installing dependency packages, or
-                      ``None`` to disable parallelization.
-    """
-
-    logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
-
-    install_executor = poetry.installation.executor.Executor(
-        env=convert_virtualenv(venv),
-        io=cleo.io.null_io.NullIO(),
-        pool=project.pool,
-        config=poetry.config.config.Config(),
-    )
-
-    installed: Set[poetry.core.packages.package.Package] = set()
-
-    def logged_install(dependency: poetry.core.packages.package.Package) -> None:
-        start = datetime.datetime.now()
-        logger.debug(f"Installing {dependency}")
-        install_executor.execute(
-            [poetry.installation.operations.install.Install(package=dependency)]
-        )
-        end = datetime.datetime.now()
-        logger.debug(f"Finished installing {dependency} in {end - start}")
-
-    with _optional_parallelize(parallels) as executor:
-        futures = []
-        for dependency in packages:
-            if dependency not in installed:
-                installed.add(dependency)
-                logger.debug(f"Queuing {dependency}")
-                future = executor(logged_install, dependency)
-                if future is not None:
-                    futures.append(future)
-            else:
-                logger.debug(f"Skipping {dependency}, already installed")
-        logger.debug("Waiting for installs to finish...")
-
-        for future in concurrent.futures.as_completed(futures):
-            # Don't actually care about the return value, just waiting on the
-            # future to ensure any exceptions that were raised in the called
-            # function are propagated.
-            future.result()
-
-
-def dedupe_packages(
-    packages: Sequence[poetry.core.packages.package.Package],
-) -> List[poetry.core.packages.package.Package]:
-    """Deduplicates a sequence of Packages while preserving ordering
-
-    Adapted from StackOverflow: https://stackoverflow.com/a/480227
-    """
-    seen: Set[poetry.core.packages.package.Package] = set()
-    # Make this faster, avoid method lookup below
-    seen_add = seen.add
-    return [item for item in packages if not (item in seen or seen_add(item))]
-
-
-def convert_virtualenv(venv: tox.tox_env.api.ToxEnv) -> poetry.utils.env.VirtualEnv:
-    """Convert a Tox venv to a Poetry venv
-
-    :param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
-    :returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
-    """
-    return poetry.utils.env.VirtualEnv(path=pathlib.Path(venv.env_dir))
-
-
-def build_package_map(project: poetry.poetry.Poetry) -> PackageMap:
-    """Build the mapping of package names to objects
-
-    :param project: Populated poetry object to load locked packages from
-    :returns: Mapping of package names to Poetry package objects
-    """
-    packages = collections.defaultdict(list)
-    for package in project.locker.locked_repository().packages:
-        packages[str(package.name)].append(package)
-
-    return packages

tox_poetry_installer/hooks/tox_add_env_config.py

@@ -1,16 +1,12 @@
 """Add required env configuration options to the tox INI file"""
-
 from typing import List
 
-import tox.config.sets
+from tox.config.sets import EnvConfigSet
-import tox.plugin
+from tox.plugin import impl
 
 
-# pylint: disable=missing-function-docstring
+@impl
-@tox.plugin.impl
+def tox_add_env_config(env_conf: EnvConfigSet):
-def tox_add_env_config(
-    env_conf: tox.config.sets.EnvConfigSet,
-):
     env_conf.add_config(
         "poetry_dep_groups",
         of_type=List[str],

tox_poetry_installer/hooks/tox_add_option.py

@@ -1,14 +1,12 @@
 """Add additional command line arguments to tox to configure plugin behavior"""
-
+from tox.config.cli.parser import ToxParser
-import tox.config.cli.parser
+from tox.plugin import impl
-import tox.plugin
 
 from tox_poetry_installer import constants
 
 
-# pylint: disable=missing-function-docstring
+@impl
-@tox.plugin.impl
+def tox_add_option(parser: ToxParser):
-def tox_add_option(parser: tox.config.cli.parser.ToxParser):
     parser.add_argument(
         "--parallel-install-threads",
         type=int,

tox_poetry_installer/hooks/tox_on_install.py

@@ -4,27 +4,40 @@ Loads the local Poetry environment and the corresponding lockfile then pulls the
 specified by the Tox environment. Finally these dependencies are installed into the Tox
 environment using the Poetry ``PipInstaller`` backend.
 """
+import collections
+import concurrent.futures
+import contextlib
+import typing
+from datetime import datetime
+from itertools import chain
+from pathlib import Path
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Sequence
+from typing import Set
 
-import itertools
+from poetry.core.packages.dependency import Dependency as PoetryDependency
-
+from poetry.core.packages.package import Package as PoetryPackage
-import tox.plugin
+from tox.plugin import impl
-import tox.tox_env.api
+from tox.tox_env.api import ToxEnv as ToxVirtualEnv
+from tox.tox_env.package import PackageToxEnv
 
+from tox_poetry_installer import constants
 from tox_poetry_installer import exceptions
 from tox_poetry_installer import logger
-from tox_poetry_installer.hooks._tox_on_install_helpers import build_package_map
+
-from tox_poetry_installer.hooks._tox_on_install_helpers import check_preconditions
+if typing.TYPE_CHECKING:
-from tox_poetry_installer.hooks._tox_on_install_helpers import convert_virtualenv
+    from tox_poetry_installer import _poetry
-from tox_poetry_installer.hooks._tox_on_install_helpers import dedupe_packages
-from tox_poetry_installer.hooks._tox_on_install_helpers import find_additional_deps
-from tox_poetry_installer.hooks._tox_on_install_helpers import find_group_deps
-from tox_poetry_installer.hooks._tox_on_install_helpers import find_project_deps
-from tox_poetry_installer.hooks._tox_on_install_helpers import install_package
 
 
-# pylint: disable=missing-function-docstring,unused-argument
+PackageMap = Dict[str, List[PoetryPackage]]
-@tox.plugin.impl
+
-def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
+
+@impl
+def tox_on_install(
+    tox_env: ToxVirtualEnv, section: str  # pylint: disable=unused-argument
+) -> None:
     try:
         poetry = check_preconditions(tox_env)
     except exceptions.SkipEnvironment as err:
@@ -41,16 +54,10 @@ def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
 
     virtualenv = convert_virtualenv(tox_env)
 
-    try:
     if not poetry.locker.is_fresh():
         logger.warning(
             f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
         )
-    except FileNotFoundError as err:
-        logger.error(f"Could not parse lockfile: {err}")
-        raise exceptions.LockfileParsingError(
-            f"Could not parse lockfile: {err}"
-        ) from err
 
     try:
         if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
@@ -62,7 +69,7 @@ def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
 
         group_deps = dedupe_packages(
             list(
-                itertools.chain(
+                chain(
                     *[
                         find_group_deps(group, packages, virtualenv, poetry)
                         for group in tox_env.conf["poetry_dep_groups"]
@@ -112,3 +119,322 @@ def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
         dependencies,
         tox_env.options.parallel_install_threads,
     )
+
+
+def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
+    """Check that the local project environment meets expectations"""
+
+    # Skip running the plugin for the provisioning environment. The provisioned environment,
+    # for alternative Tox versions and/or the ``requires`` meta dependencies is specially
+    # handled by Tox and is out of scope for this plugin. Since one of the ways to install this
+    # plugin in the first place is via the Tox provisioning environment, it quickly becomes a
+    # chicken-and-egg problem.
+    if isinstance(venv, PackageToxEnv):
+        raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
+
+    from tox_poetry_installer import _poetry
+
+    try:
+        return _poetry.Factory().create_poetry(venv.core["tox_root"])
+    # Support running the plugin when the current tox project does not use Poetry for its
+    # environment/dependency management.
+    #
+    # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
+    # is) raised in many different places for different purposes.
+    except RuntimeError as err:
+        raise exceptions.SkipEnvironment(
+            f"Skipping installation of locked dependencies due to a Poetry error: {err}"
+        ) from None
+
+
+def identify_transients(
+    dep_name: str,
+    packages: PackageMap,
+    venv: "_poetry.VirtualEnv",
+    allow_missing: Sequence[str] = (),
+) -> List[PoetryPackage]:
+    """Using a pool of packages, identify all transient dependencies of a given package name
+
+    :param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively
+                     identify the transient dependencies of
+    :param packages: All packages from the lockfile to use for identifying dependency relationships.
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param allow_missing: Sequence of package names to allow to be missing from the lockfile. Any
+                          packages that are not found in the lockfile but their name appears in this
+                          list will be silently skipped from installation.
+    :returns: List of packages that need to be installed for the requested dependency.
+
+    .. note:: The package corresponding to the dependency specified by the ``dep`` parameter will
+              be included in the returned list of packages.
+    """
+    searched: Set[str] = set()
+
+    def _transients(transient: PoetryDependency) -> List[PoetryPackage]:
+        searched.add(transient.name)
+
+        results: List[PoetryPackage] = []
+        for option in packages[transient.name]:
+            if venv.is_valid_for_marker(option.to_dependency().marker):
+                for requirement in option.requires:
+                    if requirement.name not in searched:
+                        results += _transients(requirement)
+                logger.debug(f"Including {option} for installation")
+                results.append(option)
+                break
+        else:
+            logger.debug(
+                f"Skipping {transient.name}: target python version is {'.'.join([str(item) for item in venv.get_version_info()])} but package requires {transient.marker}"
+            )
+
+        return results
+
+    try:
+        for option in packages[dep_name]:
+            if venv.is_valid_for_marker(option.to_dependency().marker):
+                dep = option.to_dependency()
+                break
+        else:
+            logger.warning(
+                f"Skipping {dep_name}: no locked version found compatible with target python version {'.'.join([str(item) for item in venv.get_version_info()])}"
+            )
+            return []
+
+        return _transients(dep)
+    except KeyError as err:
+        missing = err.args[0]
+
+        if missing in allow_missing:
+            logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
+            return []
+
+        if any(
+            delimiter in missing for delimiter in constants.PEP508_VERSION_DELIMITERS
+        ):
+            raise exceptions.LockedDepVersionConflictError(
+                f"Locked dependency '{missing}' cannot include version specifier"
+            ) from None
+
+        raise exceptions.LockedDepNotFoundError(
+            f"No version of locked dependency '{missing}' found in the project lockfile"
+        ) from None
+
+
+def find_project_deps(
+    packages: PackageMap,
+    venv: "_poetry.VirtualEnv",
+    poetry: "_poetry.Poetry",
+    extras: Sequence[str] = (),
+) -> List[PoetryPackage]:
+    """Find the root project dependencies
+
+    Recursively identify the dependencies of the root project package
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param poetry: Poetry object for the current project
+    :param extras: Sequence of extra names to include the dependencies of
+    """
+
+    required_dep_names = [
+        item.name for item in poetry.package.requires if not item.is_optional()
+    ]
+
+    extra_dep_names: List[str] = []
+    for extra in extras:
+        logger.info(f"Processing project extra '{extra}'")
+        try:
+            extra_dep_names += [item.name for item in poetry.package.extras[extra]]
+        except KeyError:
+            raise exceptions.ExtraNotFoundError(
+                f"Environment specifies project extra '{extra}' which was not found in the lockfile"
+            ) from None
+
+    dependencies: List[PoetryPackage] = []
+    for dep_name in required_dep_names + extra_dep_names:
+        dependencies += identify_transients(
+            dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
+        )
+
+    return dedupe_packages(dependencies)
+
+
+def find_additional_deps(
+    packages: PackageMap,
+    venv: "_poetry.VirtualEnv",
+    poetry: "_poetry.Poetry",
+    dep_names: Sequence[str],
+) -> List[PoetryPackage]:
+    """Find additional dependencies
+
+    Recursively identify the dependencies of an arbitrary list of package names
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param poetry: Poetry object for the current project
+    :param dep_names: Sequence of additional dependency names to recursively find the transient
+                      dependencies for
+    """
+    dependencies: List[PoetryPackage] = []
+    for dep_name in dep_names:
+        dependencies += identify_transients(
+            dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
+        )
+
+    return dedupe_packages(dependencies)
+
+
+def find_group_deps(
+    group: str,
+    packages: PackageMap,
+    venv: "_poetry.VirtualEnv",
+    poetry: "_poetry.Poetry",
+) -> List[PoetryPackage]:
+    """Find the dependencies belonging to a dependency group
+
+    Recursively identify the Poetry dev dependencies
+
+    :param group: Name of the dependency group from the project's ``pyproject.toml``
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param poetry: Poetry object for the current project
+    """
+    return find_additional_deps(
+        packages,
+        venv,
+        poetry,
+        poetry.pyproject.data["tool"]["poetry"]
+        .get("group", {})
+        .get(group, {})
+        .get("dependencies", {})
+        .keys(),
+    )
+
+
+def find_dev_deps(
+    packages: PackageMap, venv: "_poetry.VirtualEnv", poetry: "_poetry.Poetry"
+) -> List[PoetryPackage]:
+    """Find the dev dependencies
+
+    Recursively identify the Poetry dev dependencies
+
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param poetry: Poetry object for the current project
+    """
+    dev_group_deps = find_group_deps("dev", packages, venv, poetry)
+
+    # Legacy pyproject.toml poetry format:
+    legacy_dev_group_deps = find_additional_deps(
+        packages,
+        venv,
+        poetry,
+        poetry.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),
+    )
+
+    # Poetry 1.2 unions these two toml sections.
+    return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
+
+
+def install_package(
+    poetry: "_poetry.Poetry",
+    venv: ToxVirtualEnv,
+    packages: Collection["_poetry.PoetryPackage"],
+    parallels: int = 0,
+):
+    """Install a bunch of packages to a virtualenv
+
+    :param poetry: Poetry object the packages were sourced from
+    :param venv: Tox virtual environment to install the packages to
+    :param packages: List of packages to install to the virtual environment
+    :param parallels: Number of parallel processes to use for installing dependency packages, or
+                      ``None`` to disable parallelization.
+    """
+    from tox_poetry_installer import _poetry
+
+    logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
+
+    install_executor = _poetry.Executor(
+        env=convert_virtualenv(venv),
+        io=_poetry.NullIO(),
+        pool=poetry.pool,
+        config=_poetry.Config(),
+    )
+
+    installed: Set[_poetry.PoetryPackage] = set()
+
+    def logged_install(dependency: _poetry.PoetryPackage) -> None:
+        start = datetime.now()
+        logger.debug(f"Installing {dependency}")
+        install_executor.execute([_poetry.Install(package=dependency)])
+        end = datetime.now()
+        logger.debug(f"Finished installing {dependency} in {end - start}")
+
+    @contextlib.contextmanager
+    def _optional_parallelize():
+        """A bit of cheat, really
+
+        A context manager that exposes a common interface for the caller that optionally
+        enables/disables the usage of the parallel thread pooler depending on the value of
+        the ``parallels`` parameter.
+        """
+        if parallels > 0:
+            with concurrent.futures.ThreadPoolExecutor(
+                max_workers=parallels
+            ) as executor:
+                yield executor.submit
+        else:
+            yield lambda func, arg: func(arg)
+
+    with _optional_parallelize() as executor:
+        futures = []
+        for dependency in packages:
+            if dependency not in installed:
+                installed.add(dependency)
+                logger.debug(f"Queuing {dependency}")
+                future = executor(logged_install, dependency)
+                if future is not None:
+                    futures.append(future)
+            else:
+                logger.debug(f"Skipping {dependency}, already installed")
+        logger.debug("Waiting for installs to finish...")
+
+        for future in concurrent.futures.as_completed(futures):
+            # Don't actually care about the return value, just waiting on the
+            # future to ensure any exceptions that were raised in the called
+            # function are propagated.
+            future.result()
+
+
+def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
+    """Deduplicates a sequence of PoetryPackages while preserving ordering
+
+    Adapted from StackOverflow: https://stackoverflow.com/a/480227
+    """
+    seen: Set[PoetryPackage] = set()
+    # Make this faster, avoid method lookup below
+    seen_add = seen.add
+    return [p for p in packages if not (p in seen or seen_add(p))]
+
+
+def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv":
+    """Convert a Tox venv to a Poetry venv
+
+    :param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
+    :returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
+    """
+    from tox_poetry_installer import _poetry
+
+    return _poetry.VirtualEnv(path=Path(venv.env_dir))
+
+
+def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
+    """Build the mapping of package names to objects
+
+    :param poetry: Populated poetry object to load locked packages from
+    :returns: Mapping of package names to Poetry package objects
+    """
+    packages = collections.defaultdict(list)
+    for package in poetry.locker.locked_repository().packages:
+        packages[package.name].append(package)
+
+    return packages

tox_poetry_installer/logger.py

@@ -4,7 +4,6 @@ Calling ``tox.reporter.something()`` and having to format a string with the pref
 gets really old fast, but more importantly it also makes the flow of the main code
 more difficult to follow because of the added complexity.
 """
-
 import logging
 
 from tox_poetry_installer import constants