enpaul/tox-poetry-installer
1
0
Fork 0
mirror of https://github.com/enpaul/tox-poetry-installer.git synced 2025-11-03 07:39:20 +00:00
Code Issues Projects Releases Wiki Activity

Refactor dep processing to improve efficiency of installation

Browse Source 
Assemble single list of dependencies to reduce duplication and reduce installation overhead
This commit is contained in:
Ethan Paul
2020-11-12 22:14:49 -05:00
parent e37c166a8b
commit d910b6ee8d
3 changed files with 80 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tox_poetry_installer/exceptions.py
View File

@@ -5,6 +5,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
::
ToxPoetryInstallerException
+-- SkipEnvironment
+-- LockedDepVersionConflictError
+-- LockedDepNotFoundError
+-- ExtraNotFoundError
@@ -17,6 +18,10 @@ class ToxPoetryInstallerException(Exception):
"""Error while installing locked dependencies to the test environment"""
class SkipEnvironment(ToxPoetryInstallerException):
"""Current environment does not meet preconditions and should be skipped by the plugin"""
class LockedDepVersionConflictError(ToxPoetryInstallerException):
"""Locked dependencies cannot specify an alternate version for installation"""

153
tox_poetry_installer/hooks.py
View File

@@ -8,7 +8,6 @@ from typing import List
from typing import Optional
from poetry.core.packages import Package as PoetryPackage
from poetry.factory import Factory as PoetryFactory
from poetry.poetry import Poetry
from tox import hookimpl
from tox import reporter
@@ -63,116 +62,75 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
:param action: Tox action object
"""
if action.name == venv.envconfig.config.isolated_build_env:
# Skip running the plugin for the packaging environment. PEP-517 front ends can handle
# that better than we can, so let them do their thing. More to the point: if you're having
# problems in the packaging env that this plugin would solve, god help you.
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} skipping isolated build env '{action.name}'"
)
return None
try:
poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
except RuntimeError:
# Support running the plugin when the current tox project does not use Poetry for its
# environment/dependency management.
#
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
# is) raised in many different places for different purposes.
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies"
)
poetry = utilities.check_preconditions(venv, action)
except exceptions.SkipEnvironment as err:
reporter.verbosity1(str(err))
return None
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
f"{constants.REPORTER_PREFIX} Loaded project pyproject.toml from {poetry.file}"
)
package_map: PackageMap = {
package.name: package
for package in poetry.locker.locked_repository(True).packages
}
if venv.envconfig.require_locked_deps and venv.envconfig.deps:
raise exceptions.LockedDepsRequiredError(
f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
)
# Handle the installation of any locked env dependencies from the lockfile
_install_env_dependencies(venv, poetry, package_map)
package_map: PackageMap = {
package.name: package
for package in poetry.locker.locked_repository(True).packages
}
# Handle the installation of the package dependencies from the lockfile if the package is
# being installed to this venv; otherwise skip installing the package dependencies
if venv.envconfig.skip_install:
if venv.envconfig.install_dev_deps:
dev_deps: List[PoetryPackage] = [
dep
for dep in package_map.values()
if dep not in poetry.locker.locked_repository(False).packages
]
else:
dev_deps = []
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env"
)
try:
env_deps: List[PoetryPackage] = []
for dep in venv.envconfig.locked_deps:
env_deps += utilities.find_transients(package_map, dep.lower())
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
f"{constants.REPORTER_PREFIX} Identified {len(env_deps)} environment dependencies to install to env"
)
return venv.envconfig.require_locked_deps or None
if venv.envconfig.config.skipsdist:
if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
project_deps: List[PoetryPackage] = _find_project_dependencies(
venv, poetry, package_map
)
else:
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} Skipping installation of project dependencies, env does not install project package"
)
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
f"{constants.REPORTER_PREFIX} Identified {len(project_deps)} project dependencies to install to env"
)
return venv.envconfig.require_locked_deps or None
except exceptions.ToxPoetryInstallerException as err:
venv.status = "lockfile installation failed"
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
raise err
_install_project_dependencies(venv, poetry, package_map)
dependencies = list(set(dev_deps + env_deps + project_deps))
reporter.verbosity0(
f"{constants.REPORTER_PREFIX} Installing {len(dependencies)} dependencies to env '{action.name}'"
)
utilities.install_to_venv(poetry, venv, dependencies)
return venv.envconfig.require_locked_deps or None
def _install_env_dependencies(
def _find_project_dependencies(
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
):
"""Install the packages for a specified testenv
Processes the tox environment config, identifies any locked environment dependencies, pulls
them from the lockfile, and installs them to the virtual environment.
:param venv: Tox virtual environment to install the packages to
:param poetry: Poetry object the packages were sourced from
:param packages: Mapping of package names to the corresponding package object
"""
dependencies: List[PoetryPackage] = []
for dep in venv.envconfig.locked_deps:
try:
dependencies += utilities.find_transients(packages, dep.lower())
except exceptions.ToxPoetryInstallerException as err:
venv.status = "lockfile installation failed"
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
raise err
if venv.envconfig.install_dev_deps:
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} env specifies 'install_env_deps = true', including Poetry dev dependencies"
)
dev_dependencies = [
dep
for dep in poetry.locker.locked_repository(True).packages
if dep not in poetry.locker.locked_repository(False).packages
]
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dev_dependencies)} Poetry dev dependencies"
)
dependencies = list(set(dev_dependencies + dependencies))
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(venv.envconfig.locked_deps)} locked env dependencies"
)
reporter.verbosity0(
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
)
utilities.install_to_venv(poetry, venv, dependencies)
def _install_project_dependencies(
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
):
) -> List[PoetryPackage]:
"""Install the dependencies of the project package
Install all primary dependencies of the project package.
@@ -181,9 +139,6 @@ def _install_project_dependencies(
:param poetry: Poetry object the packages were sourced from
:param packages: Mapping of package names to the corresponding package object
"""
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} performing installation of project dependencies"
)
base_dependencies: List[PoetryPackage] = [
packages[item.name]
@@ -204,18 +159,6 @@ def _install_project_dependencies(
dependencies: List[PoetryPackage] = []
for dep in base_dependencies + extra_dependencies:
try:
dependencies += utilities.find_transients(packages, dep.name.lower())
except exceptions.ToxPoetryInstallerException as err:
venv.status = "lockfile installation failed"
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
raise err
dependencies += utilities.find_transients(packages, dep.name.lower())
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
)
reporter.verbosity0(
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
)
utilities.install_to_venv(poetry, venv, dependencies)
return dependencies

29
tox_poetry_installer/utilities.py
View File

@@ -4,12 +4,14 @@ from typing import Sequence
from typing import Set
from poetry.core.packages import Package as PoetryPackage
from poetry.factory import Factory as PoetryFactory
from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
from poetry.io.null_io import NullIO as PoetryNullIO
from poetry.poetry import Poetry
from poetry.puzzle.provider import Provider as PoetryProvider
from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
from tox import reporter
from tox.action import Action as ToxAction
from tox.venv import VirtualEnv as ToxVirtualEnv
from tox_poetry_installer import constants
@@ -38,7 +40,7 @@ def install_to_venv(
)
for dependency in packages:
reporter.verbosity1(f"{constants.REPORTER_PREFIX} installing {dependency}")
reporter.verbosity1(f"{constants.REPORTER_PREFIX} Installing {dependency}")
installer.install(dependency)
@@ -60,7 +62,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
def find_deps_of_deps(name: str, transients: PackageMap):
if name in PoetryProvider.UNSAFE_PACKAGES:
reporter.warning(
f"{constants.REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
f"{constants.REPORTER_PREFIX} Installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
)
else:
transients[name] = packages[name]
@@ -83,3 +85,26 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
raise exceptions.LockedDepNotFoundError(
f"No version of locked dependency '{dependency_name}' found in the project lockfile"
) from None
def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry:
"""Check that the local project environment meets expectations"""
# Skip running the plugin for the packaging environment. PEP-517 front ends can handle
# that better than we can, so let them do their thing. More to the point: if you're having
# problems in the packaging env that this plugin would solve, god help you.
if action.name == venv.envconfig.config.isolated_build_env:
raise exceptions.SkipEnvironment(
f"Skipping isolated packaging build env '{action.name}'"
)
try:
return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
# Support running the plugin when the current tox project does not use Poetry for its
# environment/dependency management.
#
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
# is) raised in many different places for different purposes.
except RuntimeError:
raise exceptions.SkipEnvironment(
"Project does not use Poetry for env management, skipping installation of locked dependencies"
) from None