mirror of
https://github.com/enpaul/tox-poetry-installer.git
synced 2025-10-28 07:00:43 +00:00
Compare commits
7 Commits
1.0.0b1
...
1d3b7834c6
| Author | SHA1 | Date | |
|---|---|---|---|
|
1d3b7834c6
|
|||
|
a270261e6c
|
|||
|
fdaee670c4
|
|||
|
62f90de297
|
|||
|
f853a4b0b7
|
|||
|
0a46b2d876
|
|||
|
c06cdfe8c2
|
@@ -227,7 +227,6 @@ error will be set to one of the "Status" values below to indicate what the error
|
||||
| `LockedDepNotFoundError` | Indicates that an item specified in the `locked_deps` config option does not match the name of a package in the Poetry lockfile. |
|
||||
| `LockedDepsRequiredError` | Indicates that a test environment with the `require_locked_deps` config option set to `true` also specified unlocked dependencies using the [`deps`](https://tox.readthedocs.io/en/latest/config.html#conf-deps) config option. |
|
||||
| `PoetryNotInstalledError` | Indicates that the `poetry` module could not be imported under the current runtime environment, and `require_poetry = true` was specified. |
|
||||
| `RequiresUnsafeDepError` | Indicates that the package-under-test depends on a package that Poetry has classified as unsafe and cannot be installed. |
|
||||
|
||||
> ℹ️ **Note:** One or more of these errors can be caused by the `pyproject.toml` being out
|
||||
> of sync with the Poetry lockfile. If this is the case, than a warning will be logged
|
||||
|
||||
1629
poetry.lock
generated
1629
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -34,13 +34,10 @@ classifiers = [
|
||||
[tool.poetry.plugins.tox]
|
||||
poetry_installer = "tox_poetry_installer"
|
||||
|
||||
[tool.poetry.extras]
|
||||
poetry = ["poetry", "cleo"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.7"
|
||||
cleo = {version = ">=1.0,<3.0", optional = true}
|
||||
poetry = {version = "^1.5.0", optional = true}
|
||||
cleo = ">=1.0,<3.0"
|
||||
poetry = "^1.5.0"
|
||||
poetry-core = "^1.1.0"
|
||||
tox = "^4"
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ try:
|
||||
from poetry.installation.operations.install import Install
|
||||
from poetry.poetry import Poetry
|
||||
from poetry.utils.env import VirtualEnv
|
||||
except ImportError:
|
||||
except ImportError as err:
|
||||
raise exceptions.PoetryNotInstalledError(
|
||||
f"No version of Poetry could be imported under the current environment for '{sys.executable}'"
|
||||
f"Failed to import a supported version of Poetry under the current environment '{sys.executable}': {err}"
|
||||
) from None
|
||||
|
||||
@@ -19,9 +19,5 @@ PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
|
||||
# console output.
|
||||
REPORTER_PREFIX: str = f"{__about__.__title__}:"
|
||||
|
||||
# Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
|
||||
# TODO: This functionality is no longer needed, should remove in a future update.
|
||||
UNSAFE_PACKAGES: Set[str] = set()
|
||||
|
||||
# Number of threads to use for installing dependencies by default
|
||||
DEFAULT_INSTALL_THREADS: int = 10
|
||||
|
||||
@@ -11,7 +11,6 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
|
||||
+-- LockedDepNotFoundError
|
||||
+-- ExtraNotFoundError
|
||||
+-- LockedDepsRequiredError
|
||||
+-- RequiresUnsafeDepError
|
||||
|
||||
"""
|
||||
|
||||
@@ -42,7 +41,3 @@ class ExtraNotFoundError(ToxPoetryInstallerException):
|
||||
|
||||
class LockedDepsRequiredError(ToxPoetryInstallerException):
|
||||
"""Environment cannot specify unlocked dependencies when locked dependencies are required"""
|
||||
|
||||
|
||||
class RequiresUnsafeDepError(ToxPoetryInstallerException):
|
||||
"""Package under test depends on an unsafe dependency and cannot be installed"""
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
"""Main hook definition module
|
||||
|
||||
All implementations of tox hooks are defined here, as well as any single-use helper functions
|
||||
specifically related to implementing the hooks (to keep the size/readability of the hook functions
|
||||
themselves manageable).
|
||||
"""
|
||||
from itertools import chain
|
||||
from typing import List
|
||||
|
||||
from tox.config.cli.parser import ToxParser
|
||||
from tox.config.sets import EnvConfigSet
|
||||
from tox.plugin import impl
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
from tox_poetry_installer import exceptions
|
||||
from tox_poetry_installer import installer
|
||||
from tox_poetry_installer import logger
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_option(parser: ToxParser):
|
||||
"""Add additional command line arguments to tox to configure plugin behavior"""
|
||||
parser.add_argument(
|
||||
"--require-poetry",
|
||||
action="store_true",
|
||||
dest="require_poetry",
|
||||
help="(deprecated) Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--parallel-install-threads",
|
||||
type=int,
|
||||
dest="parallel_install_threads",
|
||||
default=constants.DEFAULT_INSTALL_THREADS,
|
||||
help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
|
||||
)
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_env_config(env_conf: EnvConfigSet):
|
||||
"""Add required env configuration options to the tox INI file"""
|
||||
env_conf.add_config(
|
||||
"poetry_dep_groups",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of Poetry dependency groups to install to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"install_project_deps",
|
||||
of_type=bool,
|
||||
default=True,
|
||||
desc="Automatically install all Poetry primary dependencies to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_locked_deps",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Require all dependencies in the environment be installed using the Poetry lockfile",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_poetry",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"locked_deps",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of locked dependencies to install to the environment using the Poetry lockfile",
|
||||
)
|
||||
|
||||
|
||||
@impl
|
||||
def tox_on_install(
|
||||
tox_env: ToxVirtualEnv, section: str # pylint: disable=unused-argument
|
||||
) -> None:
|
||||
"""Install the dependencies for the current environment
|
||||
|
||||
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
|
||||
specified by the Tox environment. Finally these dependencies are installed into the Tox
|
||||
environment using the Poetry ``PipInstaller`` backend.
|
||||
|
||||
:param venv: Tox virtual environment object with configuration for the local Tox environment.
|
||||
:param action: Tox action object
|
||||
"""
|
||||
try:
|
||||
poetry = utilities.check_preconditions(tox_env)
|
||||
except exceptions.SkipEnvironment as err:
|
||||
if (
|
||||
isinstance(err, exceptions.PoetryNotInstalledError)
|
||||
and tox_env.conf["require_poetry"]
|
||||
):
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
logger.info(str(err))
|
||||
return
|
||||
|
||||
logger.info(f"Loaded project pyproject.toml from {poetry.file}")
|
||||
|
||||
virtualenv = utilities.convert_virtualenv(tox_env)
|
||||
|
||||
if not poetry.locker.is_fresh():
|
||||
logger.warning(
|
||||
f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
|
||||
)
|
||||
|
||||
try:
|
||||
if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
|
||||
raise exceptions.LockedDepsRequiredError(
|
||||
f"Unlocked dependencies '{tox_env.conf['deps']}' specified for environment '{tox_env.name}' which requires locked dependencies"
|
||||
)
|
||||
|
||||
packages = utilities.build_package_map(poetry)
|
||||
|
||||
group_deps = utilities.dedupe_packages(
|
||||
list(
|
||||
chain(
|
||||
*[
|
||||
utilities.find_group_deps(group, packages, virtualenv, poetry)
|
||||
for group in tox_env.conf["poetry_dep_groups"]
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(group_deps)} group dependencies to install to env"
|
||||
)
|
||||
|
||||
env_deps = utilities.find_additional_deps(
|
||||
packages, virtualenv, poetry, tox_env.conf["locked_deps"]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Identified {len(env_deps)} environment dependencies to install to env"
|
||||
)
|
||||
|
||||
# extras are not set in a testenv if skip_install=true
|
||||
try:
|
||||
extras = tox_env.conf["extras"]
|
||||
except KeyError:
|
||||
extras = []
|
||||
|
||||
if tox_env.conf["install_project_deps"]:
|
||||
project_deps = utilities.find_project_deps(
|
||||
packages, virtualenv, poetry, extras
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(project_deps)} project dependencies to install to env"
|
||||
)
|
||||
else:
|
||||
project_deps = []
|
||||
logger.info("Env does not install project package dependencies, skipping")
|
||||
except exceptions.ToxPoetryInstallerException as err:
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.error(f"Internal plugin error: {err}")
|
||||
raise err
|
||||
|
||||
dependencies = utilities.dedupe_packages(group_deps + env_deps + project_deps)
|
||||
|
||||
logger.info(f"Installing {len(dependencies)} dependencies from Poetry lock file")
|
||||
installer.install(
|
||||
poetry,
|
||||
tox_env,
|
||||
dependencies,
|
||||
tox_env.options.parallel_install_threads,
|
||||
)
|
||||
4
tox_poetry_installer/hooks/__init__.py
Normal file
4
tox_poetry_installer/hooks/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# pylint: disable=missing-module-docstring
|
||||
from tox_poetry_installer.hooks.tox_add_env_config import tox_add_env_config
|
||||
from tox_poetry_installer.hooks.tox_add_option import tox_add_option
|
||||
from tox_poetry_installer.hooks.tox_on_install import tox_on_install
|
||||
43
tox_poetry_installer/hooks/tox_add_env_config.py
Normal file
43
tox_poetry_installer/hooks/tox_add_env_config.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""Add required env configuration options to the tox INI file"""
|
||||
from typing import List
|
||||
|
||||
from tox.config.sets import EnvConfigSet
|
||||
from tox.plugin import impl
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_env_config(env_conf: EnvConfigSet):
|
||||
env_conf.add_config(
|
||||
"poetry_dep_groups",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of Poetry dependency groups to install to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"install_project_deps",
|
||||
of_type=bool,
|
||||
default=True,
|
||||
desc="Automatically install all Poetry primary dependencies to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_locked_deps",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Require all dependencies in the environment be installed using the Poetry lockfile",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_poetry",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"locked_deps",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of locked dependencies to install to the environment using the Poetry lockfile",
|
||||
)
|
||||
16
tox_poetry_installer/hooks/tox_add_option.py
Normal file
16
tox_poetry_installer/hooks/tox_add_option.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""Add additional command line arguments to tox to configure plugin behavior"""
|
||||
from tox.config.cli.parser import ToxParser
|
||||
from tox.plugin import impl
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_option(parser: ToxParser):
|
||||
parser.add_argument(
|
||||
"--parallel-install-threads",
|
||||
type=int,
|
||||
dest="parallel_install_threads",
|
||||
default=constants.DEFAULT_INSTALL_THREADS,
|
||||
help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
|
||||
)
|
||||
@@ -1,10 +1,17 @@
|
||||
"""Helper utility functions, usually bridging Tox and Poetry functionality"""
|
||||
# Silence this one globally to support the internal function imports for the proxied poetry module.
|
||||
# See the docstring in 'tox_poetry_installer._poetry' for more context.
|
||||
# pylint: disable=import-outside-toplevel
|
||||
"""Install the dependencies for the current environment
|
||||
|
||||
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
|
||||
specified by the Tox environment. Finally these dependencies are installed into the Tox
|
||||
environment using the Poetry ``PipInstaller`` backend.
|
||||
"""
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import typing
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Collection
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Sequence
|
||||
@@ -12,6 +19,7 @@ from typing import Set
|
||||
|
||||
from poetry.core.packages.dependency import Dependency as PoetryDependency
|
||||
from poetry.core.packages.package import Package as PoetryPackage
|
||||
from tox.plugin import impl
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
from tox.tox_env.package import PackageToxEnv
|
||||
|
||||
@@ -26,6 +34,93 @@ if typing.TYPE_CHECKING:
|
||||
PackageMap = Dict[str, List[PoetryPackage]]
|
||||
|
||||
|
||||
@impl
|
||||
def tox_on_install(
|
||||
tox_env: ToxVirtualEnv, section: str # pylint: disable=unused-argument
|
||||
) -> None:
|
||||
try:
|
||||
poetry = check_preconditions(tox_env)
|
||||
except exceptions.SkipEnvironment as err:
|
||||
if (
|
||||
isinstance(err, exceptions.PoetryNotInstalledError)
|
||||
and tox_env.conf["require_poetry"]
|
||||
):
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
logger.info(str(err))
|
||||
return
|
||||
|
||||
logger.info(f"Loaded project pyproject.toml from {poetry.file}")
|
||||
|
||||
virtualenv = convert_virtualenv(tox_env)
|
||||
|
||||
if not poetry.locker.is_fresh():
|
||||
logger.warning(
|
||||
f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
|
||||
)
|
||||
|
||||
try:
|
||||
if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
|
||||
raise exceptions.LockedDepsRequiredError(
|
||||
f"Unlocked dependencies '{tox_env.conf['deps']}' specified for environment '{tox_env.name}' which requires locked dependencies"
|
||||
)
|
||||
|
||||
packages = build_package_map(poetry)
|
||||
|
||||
group_deps = dedupe_packages(
|
||||
list(
|
||||
chain(
|
||||
*[
|
||||
find_group_deps(group, packages, virtualenv, poetry)
|
||||
for group in tox_env.conf["poetry_dep_groups"]
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(group_deps)} group dependencies to install to env"
|
||||
)
|
||||
|
||||
env_deps = find_additional_deps(
|
||||
packages, virtualenv, poetry, tox_env.conf["locked_deps"]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Identified {len(env_deps)} environment dependencies to install to env"
|
||||
)
|
||||
|
||||
# extras are not set in a testenv if skip_install=true
|
||||
try:
|
||||
extras = tox_env.conf["extras"]
|
||||
except KeyError:
|
||||
extras = []
|
||||
|
||||
if tox_env.conf["install_project_deps"]:
|
||||
project_deps = find_project_deps(packages, virtualenv, poetry, extras)
|
||||
logger.info(
|
||||
f"Identified {len(project_deps)} project dependencies to install to env"
|
||||
)
|
||||
else:
|
||||
project_deps = []
|
||||
logger.info("Env does not install project package dependencies, skipping")
|
||||
except exceptions.ToxPoetryInstallerException as err:
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.error(f"Internal plugin error: {err}")
|
||||
raise err
|
||||
|
||||
dependencies = dedupe_packages(group_deps + env_deps + project_deps)
|
||||
|
||||
logger.info(f"Installing {len(dependencies)} dependencies from Poetry lock file")
|
||||
install_package(
|
||||
poetry,
|
||||
tox_env,
|
||||
dependencies,
|
||||
tox_env.options.parallel_install_threads,
|
||||
)
|
||||
|
||||
|
||||
def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
|
||||
"""Check that the local project environment meets expectations"""
|
||||
|
||||
@@ -37,13 +132,6 @@ def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
|
||||
if isinstance(venv, PackageToxEnv):
|
||||
raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
|
||||
|
||||
if venv.options.require_poetry:
|
||||
logger.warning(
|
||||
"DEPRECATION: The '--require-poetry' runtime option is deprecated and will be "
|
||||
"removed in version 1.0.0. Please update test environments that require Poetry to "
|
||||
"set the 'require_poetry = true' option in tox.ini"
|
||||
)
|
||||
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
try:
|
||||
@@ -53,36 +141,12 @@ def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
|
||||
#
|
||||
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
|
||||
# is) raised in many different places for different purposes.
|
||||
except RuntimeError:
|
||||
except RuntimeError as err:
|
||||
raise exceptions.SkipEnvironment(
|
||||
"Project does not use Poetry for env management, skipping installation of locked dependencies"
|
||||
f"Skipping installation of locked dependencies due to a Poetry error: {err}"
|
||||
) from None
|
||||
|
||||
|
||||
def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv":
|
||||
"""Convert a Tox venv to a Poetry venv
|
||||
|
||||
:param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
|
||||
:returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
return _poetry.VirtualEnv(path=Path(venv.env_dir))
|
||||
|
||||
|
||||
def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
|
||||
"""Build the mapping of package names to objects
|
||||
|
||||
:param poetry: Populated poetry object to load locked packages from
|
||||
:returns: Mapping of package names to Poetry package objects
|
||||
"""
|
||||
packages = collections.defaultdict(list)
|
||||
for package in poetry.locker.locked_repository().packages:
|
||||
packages[package.name].append(package)
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def identify_transients(
|
||||
dep_name: str,
|
||||
packages: PackageMap,
|
||||
@@ -139,13 +203,6 @@ def identify_transients(
|
||||
except KeyError as err:
|
||||
missing = err.args[0]
|
||||
|
||||
if missing in constants.UNSAFE_PACKAGES:
|
||||
logger.warning(
|
||||
f"Installing package '{missing}' using Poetry is not supported and will be skipped"
|
||||
)
|
||||
logger.debug(f"Skipping {missing}: designated unsafe by Poetry")
|
||||
return []
|
||||
|
||||
if missing in allow_missing:
|
||||
logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
|
||||
return []
|
||||
@@ -178,11 +235,6 @@ def find_project_deps(
|
||||
:param extras: Sequence of extra names to include the dependencies of
|
||||
"""
|
||||
|
||||
if any(dep.name in constants.UNSAFE_PACKAGES for dep in poetry.package.requires):
|
||||
raise exceptions.RequiresUnsafeDepError(
|
||||
f"Project package requires one or more unsafe dependencies ({', '.join(constants.UNSAFE_PACKAGES)}) which cannot be installed with Poetry"
|
||||
)
|
||||
|
||||
required_dep_names = [
|
||||
item.name for item in poetry.package.requires if not item.is_optional()
|
||||
]
|
||||
@@ -283,6 +335,76 @@ def find_dev_deps(
|
||||
return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
|
||||
|
||||
|
||||
def install_package(
|
||||
poetry: "_poetry.Poetry",
|
||||
venv: ToxVirtualEnv,
|
||||
packages: Collection["_poetry.PoetryPackage"],
|
||||
parallels: int = 0,
|
||||
):
|
||||
"""Install a bunch of packages to a virtualenv
|
||||
|
||||
:param poetry: Poetry object the packages were sourced from
|
||||
:param venv: Tox virtual environment to install the packages to
|
||||
:param packages: List of packages to install to the virtual environment
|
||||
:param parallels: Number of parallel processes to use for installing dependency packages, or
|
||||
``None`` to disable parallelization.
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
|
||||
|
||||
install_executor = _poetry.Executor(
|
||||
env=convert_virtualenv(venv),
|
||||
io=_poetry.NullIO(),
|
||||
pool=poetry.pool,
|
||||
config=_poetry.Config(),
|
||||
)
|
||||
|
||||
installed: Set[_poetry.PoetryPackage] = set()
|
||||
|
||||
def logged_install(dependency: _poetry.PoetryPackage) -> None:
|
||||
start = datetime.now()
|
||||
logger.debug(f"Installing {dependency}")
|
||||
install_executor.execute([_poetry.Install(package=dependency)])
|
||||
end = datetime.now()
|
||||
logger.debug(f"Finished installing {dependency} in {end - start}")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _optional_parallelize():
|
||||
"""A bit of cheat, really
|
||||
|
||||
A context manager that exposes a common interface for the caller that optionally
|
||||
enables/disables the usage of the parallel thread pooler depending on the value of
|
||||
the ``parallels`` parameter.
|
||||
"""
|
||||
if parallels > 0:
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=parallels
|
||||
) as executor:
|
||||
yield executor.submit
|
||||
else:
|
||||
yield lambda func, arg: func(arg)
|
||||
|
||||
with _optional_parallelize() as executor:
|
||||
futures = []
|
||||
for dependency in packages:
|
||||
if dependency not in installed:
|
||||
installed.add(dependency)
|
||||
logger.debug(f"Queuing {dependency}")
|
||||
future = executor(logged_install, dependency)
|
||||
if future is not None:
|
||||
futures.append(future)
|
||||
else:
|
||||
logger.debug(f"Skipping {dependency}, already installed")
|
||||
logger.debug("Waiting for installs to finish...")
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Don't actually care about the return value, just waiting on the
|
||||
# future to ensure any exceptions that were raised in the called
|
||||
# function are propagated.
|
||||
future.result()
|
||||
|
||||
|
||||
def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
|
||||
"""Deduplicates a sequence of PoetryPackages while preserving ordering
|
||||
|
||||
@@ -292,3 +414,27 @@ def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
|
||||
# Make this faster, avoid method lookup below
|
||||
seen_add = seen.add
|
||||
return [p for p in packages if not (p in seen or seen_add(p))]
|
||||
|
||||
|
||||
def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv":
|
||||
"""Convert a Tox venv to a Poetry venv
|
||||
|
||||
:param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
|
||||
:returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
return _poetry.VirtualEnv(path=Path(venv.env_dir))
|
||||
|
||||
|
||||
def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
|
||||
"""Build the mapping of package names to objects
|
||||
|
||||
:param poetry: Populated poetry object to load locked packages from
|
||||
:returns: Mapping of package names to Poetry package objects
|
||||
"""
|
||||
packages = collections.defaultdict(list)
|
||||
for package in poetry.locker.locked_repository().packages:
|
||||
packages[package.name].append(package)
|
||||
|
||||
return packages
|
||||
@@ -1,88 +0,0 @@
|
||||
"""Funcationality for performing virtualenv installation"""
|
||||
# Silence this one globally to support the internal function imports for the proxied poetry module.
|
||||
# See the docstring in 'tox_poetry_installer._poetry' for more context.
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import typing
|
||||
from datetime import datetime
|
||||
from typing import Collection
|
||||
from typing import Set
|
||||
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
|
||||
from tox_poetry_installer import logger
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
|
||||
def install(
|
||||
poetry: "_poetry.Poetry",
|
||||
venv: ToxVirtualEnv,
|
||||
packages: Collection["_poetry.PoetryPackage"],
|
||||
parallels: int = 0,
|
||||
):
|
||||
"""Install a bunch of packages to a virtualenv
|
||||
|
||||
:param poetry: Poetry object the packages were sourced from
|
||||
:param venv: Tox virtual environment to install the packages to
|
||||
:param packages: List of packages to install to the virtual environment
|
||||
:param parallels: Number of parallel processes to use for installing dependency packages, or
|
||||
``None`` to disable parallelization.
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
|
||||
|
||||
install_executor = _poetry.Executor(
|
||||
env=utilities.convert_virtualenv(venv),
|
||||
io=_poetry.NullIO(),
|
||||
pool=poetry.pool,
|
||||
config=_poetry.Config(),
|
||||
)
|
||||
|
||||
installed: Set[_poetry.PoetryPackage] = set()
|
||||
|
||||
def logged_install(dependency: _poetry.PoetryPackage) -> None:
|
||||
start = datetime.now()
|
||||
logger.debug(f"Installing {dependency}")
|
||||
install_executor.execute([_poetry.Install(package=dependency)])
|
||||
end = datetime.now()
|
||||
logger.debug(f"Finished installing {dependency} in {end - start}")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _optional_parallelize():
|
||||
"""A bit of cheat, really
|
||||
|
||||
A context manager that exposes a common interface for the caller that optionally
|
||||
enables/disables the usage of the parallel thread pooler depending on the value of
|
||||
the ``parallels`` parameter.
|
||||
"""
|
||||
if parallels > 0:
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=parallels
|
||||
) as executor:
|
||||
yield executor.submit
|
||||
else:
|
||||
yield lambda func, arg: func(arg)
|
||||
|
||||
with _optional_parallelize() as executor:
|
||||
futures = []
|
||||
for dependency in packages:
|
||||
if dependency not in installed:
|
||||
installed.add(dependency)
|
||||
logger.debug(f"Queuing {dependency}")
|
||||
future = executor(logged_install, dependency)
|
||||
if future is not None:
|
||||
futures.append(future)
|
||||
else:
|
||||
logger.debug(f"Skipping {dependency}, already installed")
|
||||
logger.debug("Waiting for installs to finish...")
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Don't actually care about the return value, just waiting on the
|
||||
# future to ensure any exceptions that were raised in the called
|
||||
# function are propagated.
|
||||
future.result()
|
||||
Reference in New Issue
Block a user